evalt 0.8.5__tar.gz

evalt-0.8.5/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jonathan Larson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

evalt-0.8.5/PKG-INFO

@@ -0,0 +1,264 @@
+Metadata-Version: 2.4
+Name: evalt
+Version: 0.8.5
+Summary: A durable LLM router that finds the most accurate configuration at your price.
+Author: Jonathan Larson
+Maintainer: Jonathan Larson
+License-Expression: MIT
+Project-URL: Homepage, https://evalt.dev
+Project-URL: Documentation, https://evalt.dev/#sdk
+Project-URL: Changelog, https://evalt.dev/#sdk
+Keywords: llm,prompt-engineering,evaluation,model-routing,ci
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# Evalt Python SDK
+
+Evalt is a durable runtime router. Your application gives it a prompt, an input, and a
+stable route name. Evalt finds the lowest-cost prompt/model/reasoning configuration that
+clears the approved validation target (95% by default), records the decision in SQLite,
+and uses a separate capped test budget when traffic, a new model, or a provider-price
+change makes retesting worthwhile.
+
+## Install
+
+After the public package is released:
+
+```bash
+python -m pip install evalt
+```
+
+Until that account-bound publication is complete, install the verified versioned wheel
+from this repository checkout:
+
+```bash
+python -m venv .venv
+python -m pip install dist/evalt-0.8.5-py3-none-any.whl
+evalt --version
+```
+
+`evalt` is the primary import and command. `modelsieve`, `last_good_prompt`, and `lgp`
+remain compatibility aliases for existing integrations.
+
+The SDK is MIT licensed. Hosted Evalt Pro is a separate managed service; using the free
+package never creates a platform charge.
+
+## Production API
+
+```python
+from evalt import Evalt
+
+evalt = Evalt(api_key=OPENROUTER_API_KEY)
+answer = evalt.run(
+    "Classify this support request as billing, account, or technical.",
+    ticket,
+    route="support-routing",
+    test_budget_usd="auto",
+)
+
+send(answer.content)
+answer.accept()                 # or answer.correct("billing")
+print(evalt.route_status("support-routing"))
+```
+
+The first call uses the selected bootstrap route within the production price ceiling.
+Explicit feedback becomes the route-specific evaluation set. On a later call, Evalt can
+launch a background maintenance run after the configured evidence/traffic threshold is
+met. Automatic test spend is bounded by `max_test_budget_usd` (USD 1 by default); a
+retest never spends from an unlimited hidden allowance.
+
+The focused default is `objective="lowest_cost_at_accuracy"` with
+`target_accuracy=0.95`: Evalt promotes the cheapest tested configuration that clears that
+approved bar. A price-first frontier and an incumbent-preservation migration mode remain
+available:
+
+```python
+answer = evalt.run(
+    prompt,
+    input,
+    route="support-routing",
+    price_usd=0.05,
+    target_accuracy=0.97,
+    objective="lowest_cost_at_accuracy",  # or "best_within_price"
+    test_budget_usd=0.75,
+)
+```
+
+`incumbent_model` is optional. Use it with
+`objective="match_baseline_at_lowest_cost"` when migrating an existing workflow and you
+specifically want the incumbent's measured validation quality to be the bar. New
+workflows need no comparison model: their approved validation target is the bar.
+
+Reasoning effort is tested as part of the model configuration only when the current ZDR
+endpoint supports it. The adaptive search first runs one configuration across a broad
+price/intelligence frontier, then spends the remaining test budget on effort variants for
+models in the observed task-capability band. Reasoning configurations receive larger,
+budget-reserved completion headroom so internal reasoning cannot silently consume the
+requested visible response size. A transient empty response gets one budget-checked
+retry. An explicitly truncated response fails that configuration immediately: Evalt does
+not double a costly reasoning allowance after the provider has already hit the bounded
+limit. Production cost uses the route's
+90th-percentile approved input and output lengths. The default quality floor is 95%, and
+held-out cases are repeated twice before promotion. A measured 100% means every repeated
+approved final-test scenario passed on every configured repeat; it is not a guarantee
+about every future input. Reports show distinct scenario count and execution count
+separately.
+
+Model roles are selected separately:
+
+- the test designer / prompt improver uses the cheapest catalog model near the top of the
+  current intelligence range;
+- the judge may use a lower-cost model only after route-specific verdict calibration;
+- production targets come from the price/intelligence frontier, then win or lose on the
+  route's frozen human-approved cases.
+
+Higher maintenance budgets tighten the intelligence floors and broaden the target field.
+Catalog benchmarks shortlist contenders only; they never promote a route without the
+task-specific holdout.
+
+## Explicit optimization and CI
+
+```bash
+evalt init evalt.json
+evalt validate evalt.json
+export OPENROUTER_API_KEY="..."
+evalt optimize evalt.json --output evalt-result.json
+evalt check evalt-result.json --min-pass-rate 0.95
+```
+
+`init`, `validate`, and `check` are offline and make no provider calls. `optimize` uses the
+single `max_optimization_cost_usd` value in the suite as a hard cap across optimization,
+target runs, judging, and every selected model. The command reports partial coverage
+instead of calling an unfinished tournament globally best. It emits line-delimited
+progress events to stderr as each model starts, finishes, or fails, while the final JSON
+stays on stdout and at the requested output path. Model/scenario concurrency and the
+wall-clock timeout for one provider response can be overridden per run.
+
+Use stricter CI gates when needed:
+
+```bash
+evalt check evalt-result.json \
+  --min-pass-rate 0.95 \
+  --max-cost-per-success 0.002 \
+  --require-complete-coverage
+```
+
+The command exits `0` on pass, `1` when the measured result fails the gate, and `2` for an
+invalid file or runtime error.
+
+To inspect the CI contract without a provider call:
+
+```bash
+evalt check examples/passing-result.json --min-pass-rate 0.95 --require-complete-coverage
+```
+
+## Explicit suite API
+
+```python
+from evalt import Evalt, Suite
+
+suite = Suite.load("evalt.json")  # validates without a provider call
+result = Evalt().run(suite)
+
+print(result.winner.model)
+print(result.winner.selected_prompt)
+print(result.winner.holdout_pass_rate)
+print(result.winner.estimated_cost_per_successful_call_usd)
+
+result.save("evalt-result.json")
+result.save_regression_suite("evalt-regression.json")
+```
+
+For lower-level integrations, `Client.optimize(...)` remains available. `Suite` is the
+recommended surface because the full evaluation contract stays inspectable, serializable,
+and offline-validatable before spend.
+
+## Suite shape
+
+```json
+{
+  "schema": "evalt-suite-v1",
+  "name": "support-routing",
+  "prompt": "Classify the support message. Return one route label.",
+  "examples": [
+    {"id": "billing-1", "input": "I was charged twice", "approved_output": "billing"},
+    {"id": "account-1", "input": "My reset link expired", "approved_output": "account"},
+    {"id": "technical-1", "input": "The app freezes", "approved_output": "technical"}
+  ],
+  "models": ["qwen/qwen3.5-9b", "google/gemini-3-flash-preview"],
+  "optimizer_model": "openai/gpt-5.6-luna",
+  "evaluator_model": "openai/gpt-5.6-luna",
+  "evaluator": {"type": "semantic"},
+  "quality_threshold": 0.95,
+  "max_optimization_cost_usd": 2.0,
+  "rounds": 3,
+  "max_parallel_models": 8,
+  "max_parallel_scenarios": 16,
+  "allow_few_shot": true,
+  "max_few_shot_examples": 3
+}
+```
+
+With the default 20% final-test split, use at least 25 approved scenarios to obtain the
+minimum five distinct final-test scenarios for a non-exploratory result. Repeats measure
+consistency; they never inflate the distinct scenario count. A scenario may contain
+a `turns` array for multi-turn behavior; Evalt keeps the whole conversation in one split
+and replays prior assistant context. Few-shot examples can come only from the training
+split and are removed while evaluating their own scenario.
+
+For exact outputs, replace the semantic evaluator with a deterministic contract:
+
+```json
+"evaluator": {
+  "type": "exact_json",
+  "required_keys": ["x", "y"],
+  "allow_additional_properties": false,
+  "normalize_rational_strings": true
+}
+```
+
+This makes no evaluator-model call. `exact_text` is also available for strict labels.
+Independent model lanes run concurrently (eight by default) and each lane evaluates up
+to sixteen independent case executions concurrently. Model lanes are configurable up to
+16 and case execution concurrency up to 64. Repeated executions are parallel work units;
+turns inside one multi-turn scenario remain ordered. Every in-flight estimate is reserved
+against the one hard suite budget. A prompt that already scores 100% on the validation
+split skips training replay and rewriting and moves directly to the frozen final test.
+
+## Provider and data contract
+
+- The API key is read from `OPENROUTER_API_KEY`; it is never written to a suite or result.
+- Every OpenRouter request requires Zero Data Retention and denies provider data collection.
+- Current provider pricing is refreshed at least hourly by default before calls; a changed
+  price changes the catalog revision and makes the durable route due for a bounded re-test.
+  An unpriced or unbounded route fails closed. Set `catalog_ttl_seconds` on
+  `OpenRouterTransport` when a different refresh interval is required.
+- Exact provider-reported cost is accumulated in the result.
+- The SDK adds no platform or usage fee. Hosted BYOK is also free during early access.
+  A future paid hosted control plane for shared history, CI, scheduled Model Watch,
+  permissions, managed credentials, and support is an evidence-gated hypothesis rather
+  than a current subscription offer.
+
+## Compatibility
+
+Evalt follows semantic versioning for the primary `evalt` Python API, CLI command names,
+suite schema, result schema, and exit-code contract. Deprecations remain available for
+at least one minor release and are documented before removal. The legacy
+`modelsieve`/`last_good_prompt` imports and `lgp` command are compatibility shims, not the
+recommended surface for new integrations.
+
+An exported result is evidence about its frozen examples and named model versions—not a
+general intelligence ranking or a promise about unseen production traffic.

evalt-0.8.5/README.md

@@ -0,0 +1,237 @@
+# Evalt Python SDK
+
+Evalt is a durable runtime router. Your application gives it a prompt, an input, and a
+stable route name. Evalt finds the lowest-cost prompt/model/reasoning configuration that
+clears the approved validation target (95% by default), records the decision in SQLite,
+and uses a separate capped test budget when traffic, a new model, or a provider-price
+change makes retesting worthwhile.
+
+## Install
+
+After the public package is released:
+
+```bash
+python -m pip install evalt
+```
+
+Until that account-bound publication is complete, install the verified versioned wheel
+from this repository checkout:
+
+```bash
+python -m venv .venv
+python -m pip install dist/evalt-0.8.5-py3-none-any.whl
+evalt --version
+```
+
+`evalt` is the primary import and command. `modelsieve`, `last_good_prompt`, and `lgp`
+remain compatibility aliases for existing integrations.
+
+The SDK is MIT licensed. Hosted Evalt Pro is a separate managed service; using the free
+package never creates a platform charge.
+
+## Production API
+
+```python
+from evalt import Evalt
+
+evalt = Evalt(api_key=OPENROUTER_API_KEY)
+answer = evalt.run(
+    "Classify this support request as billing, account, or technical.",
+    ticket,
+    route="support-routing",
+    test_budget_usd="auto",
+)
+
+send(answer.content)
+answer.accept()                 # or answer.correct("billing")
+print(evalt.route_status("support-routing"))
+```
+
+The first call uses the selected bootstrap route within the production price ceiling.
+Explicit feedback becomes the route-specific evaluation set. On a later call, Evalt can
+launch a background maintenance run after the configured evidence/traffic threshold is
+met. Automatic test spend is bounded by `max_test_budget_usd` (USD 1 by default); a
+retest never spends from an unlimited hidden allowance.
+
+The focused default is `objective="lowest_cost_at_accuracy"` with
+`target_accuracy=0.95`: Evalt promotes the cheapest tested configuration that clears that
+approved bar. A price-first frontier and an incumbent-preservation migration mode remain
+available:
+
+```python
+answer = evalt.run(
+    prompt,
+    input,
+    route="support-routing",
+    price_usd=0.05,
+    target_accuracy=0.97,
+    objective="lowest_cost_at_accuracy",  # or "best_within_price"
+    test_budget_usd=0.75,
+)
+```
+
+`incumbent_model` is optional. Use it with
+`objective="match_baseline_at_lowest_cost"` when migrating an existing workflow and you
+specifically want the incumbent's measured validation quality to be the bar. New
+workflows need no comparison model: their approved validation target is the bar.
+
+Reasoning effort is tested as part of the model configuration only when the current ZDR
+endpoint supports it. The adaptive search first runs one configuration across a broad
+price/intelligence frontier, then spends the remaining test budget on effort variants for
+models in the observed task-capability band. Reasoning configurations receive larger,
+budget-reserved completion headroom so internal reasoning cannot silently consume the
+requested visible response size. A transient empty response gets one budget-checked
+retry. An explicitly truncated response fails that configuration immediately: Evalt does
+not double a costly reasoning allowance after the provider has already hit the bounded
+limit. Production cost uses the route's
+90th-percentile approved input and output lengths. The default quality floor is 95%, and
+held-out cases are repeated twice before promotion. A measured 100% means every repeated
+approved final-test scenario passed on every configured repeat; it is not a guarantee
+about every future input. Reports show distinct scenario count and execution count
+separately.
+
+Model roles are selected separately:
+
+- the test designer / prompt improver uses the cheapest catalog model near the top of the
+  current intelligence range;
+- the judge may use a lower-cost model only after route-specific verdict calibration;
+- production targets come from the price/intelligence frontier, then win or lose on the
+  route's frozen human-approved cases.
+
+Higher maintenance budgets tighten the intelligence floors and broaden the target field.
+Catalog benchmarks shortlist contenders only; they never promote a route without the
+task-specific holdout.
+
+## Explicit optimization and CI
+
+```bash
+evalt init evalt.json
+evalt validate evalt.json
+export OPENROUTER_API_KEY="..."
+evalt optimize evalt.json --output evalt-result.json
+evalt check evalt-result.json --min-pass-rate 0.95
+```
+
+`init`, `validate`, and `check` are offline and make no provider calls. `optimize` uses the
+single `max_optimization_cost_usd` value in the suite as a hard cap across optimization,
+target runs, judging, and every selected model. The command reports partial coverage
+instead of calling an unfinished tournament globally best. It emits line-delimited
+progress events to stderr as each model starts, finishes, or fails, while the final JSON
+stays on stdout and at the requested output path. Model/scenario concurrency and the
+wall-clock timeout for one provider response can be overridden per run.
+
+Use stricter CI gates when needed:
+
+```bash
+evalt check evalt-result.json \
+  --min-pass-rate 0.95 \
+  --max-cost-per-success 0.002 \
+  --require-complete-coverage
+```
+
+The command exits `0` on pass, `1` when the measured result fails the gate, and `2` for an
+invalid file or runtime error.
+
+To inspect the CI contract without a provider call:
+
+```bash
+evalt check examples/passing-result.json --min-pass-rate 0.95 --require-complete-coverage
+```
+
+## Explicit suite API
+
+```python
+from evalt import Evalt, Suite
+
+suite = Suite.load("evalt.json")  # validates without a provider call
+result = Evalt().run(suite)
+
+print(result.winner.model)
+print(result.winner.selected_prompt)
+print(result.winner.holdout_pass_rate)
+print(result.winner.estimated_cost_per_successful_call_usd)
+
+result.save("evalt-result.json")
+result.save_regression_suite("evalt-regression.json")
+```
+
+For lower-level integrations, `Client.optimize(...)` remains available. `Suite` is the
+recommended surface because the full evaluation contract stays inspectable, serializable,
+and offline-validatable before spend.
+
+## Suite shape
+
+```json
+{
+  "schema": "evalt-suite-v1",
+  "name": "support-routing",
+  "prompt": "Classify the support message. Return one route label.",
+  "examples": [
+    {"id": "billing-1", "input": "I was charged twice", "approved_output": "billing"},
+    {"id": "account-1", "input": "My reset link expired", "approved_output": "account"},
+    {"id": "technical-1", "input": "The app freezes", "approved_output": "technical"}
+  ],
+  "models": ["qwen/qwen3.5-9b", "google/gemini-3-flash-preview"],
+  "optimizer_model": "openai/gpt-5.6-luna",
+  "evaluator_model": "openai/gpt-5.6-luna",
+  "evaluator": {"type": "semantic"},
+  "quality_threshold": 0.95,
+  "max_optimization_cost_usd": 2.0,
+  "rounds": 3,
+  "max_parallel_models": 8,
+  "max_parallel_scenarios": 16,
+  "allow_few_shot": true,
+  "max_few_shot_examples": 3
+}
+```
+
+With the default 20% final-test split, use at least 25 approved scenarios to obtain the
+minimum five distinct final-test scenarios for a non-exploratory result. Repeats measure
+consistency; they never inflate the distinct scenario count. A scenario may contain
+a `turns` array for multi-turn behavior; Evalt keeps the whole conversation in one split
+and replays prior assistant context. Few-shot examples can come only from the training
+split and are removed while evaluating their own scenario.
+
+For exact outputs, replace the semantic evaluator with a deterministic contract:
+
+```json
+"evaluator": {
+  "type": "exact_json",
+  "required_keys": ["x", "y"],
+  "allow_additional_properties": false,
+  "normalize_rational_strings": true
+}
+```
+
+This makes no evaluator-model call. `exact_text` is also available for strict labels.
+Independent model lanes run concurrently (eight by default) and each lane evaluates up
+to sixteen independent case executions concurrently. Model lanes are configurable up to
+16 and case execution concurrency up to 64. Repeated executions are parallel work units;
+turns inside one multi-turn scenario remain ordered. Every in-flight estimate is reserved
+against the one hard suite budget. A prompt that already scores 100% on the validation
+split skips training replay and rewriting and moves directly to the frozen final test.
+
+## Provider and data contract
+
+- The API key is read from `OPENROUTER_API_KEY`; it is never written to a suite or result.
+- Every OpenRouter request requires Zero Data Retention and denies provider data collection.
+- Current provider pricing is refreshed at least hourly by default before calls; a changed
+  price changes the catalog revision and makes the durable route due for a bounded re-test.
+  An unpriced or unbounded route fails closed. Set `catalog_ttl_seconds` on
+  `OpenRouterTransport` when a different refresh interval is required.
+- Exact provider-reported cost is accumulated in the result.
+- The SDK adds no platform or usage fee. Hosted BYOK is also free during early access.
+  A future paid hosted control plane for shared history, CI, scheduled Model Watch,
+  permissions, managed credentials, and support is an evidence-gated hypothesis rather
+  than a current subscription offer.
+
+## Compatibility
+
+Evalt follows semantic versioning for the primary `evalt` Python API, CLI command names,
+suite schema, result schema, and exit-code contract. Deprecations remain available for
+at least one minor release and are documented before removal. The legacy
+`modelsieve`/`last_good_prompt` imports and `lgp` command are compatibility shims, not the
+recommended surface for new integrations.
+
+An exported result is evidence about its frozen examples and named model versions—not a
+general intelligence ranking or a promise about unseen production traffic.

evalt-0.8.5/pyproject.toml

@@ -0,0 +1,48 @@
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "evalt"
+version = "0.8.5"
+description = "A durable LLM router that finds the most accurate configuration at your price."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Jonathan Larson" }]
+maintainers = [{ name = "Jonathan Larson" }]
+keywords = ["llm", "prompt-engineering", "evaluation", "model-routing", "ci"]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Environment :: Console",
+  "Intended Audience :: Developers",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Software Development :: Testing",
+]
+dependencies = []
+
+[project.urls]
+Homepage = "https://evalt.dev"
+Documentation = "https://evalt.dev/#sdk"
+Changelog = "https://evalt.dev/#sdk"
+
+[project.scripts]
+evalt = "evalt.cli:main"
+modelsieve = "modelsieve.cli:main"
+lgp = "last_good_prompt.cli:main"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+evalt = ["py.typed"]

evalt-0.8.5/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

evalt-0.8.5/src/evalt/__init__.py

@@ -0,0 +1,39 @@
+"""Public SDK surface for Evalt."""
+
+from .core import (
+    BudgetExceeded,
+    Client,
+    DraftAnswer,
+    Evalt,
+    Example,
+    GateReport,
+    ModelResult,
+    OptimizationResult,
+    ProviderError,
+    RolePlan,
+    RoutedAnswer,
+    Suite,
+    Turn,
+    check_result,
+    select_role_plan,
+)
+
+__all__ = [
+    "BudgetExceeded",
+    "Client",
+    "DraftAnswer",
+    "Evalt",
+    "Example",
+    "GateReport",
+    "ModelResult",
+    "OptimizationResult",
+    "ProviderError",
+    "RolePlan",
+    "RoutedAnswer",
+    "Suite",
+    "Turn",
+    "check_result",
+    "select_role_plan",
+]
+
+__version__ = "0.8.5"

evalt-0.8.5/src/evalt/__main__.py

@@ -0,0 +1,5 @@
+"""Support ``python -m evalt``."""
+
+from .cli import main
+
+raise SystemExit(main())