strop-cli 0.1.0__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {strop_cli-0.1.0 → strop_cli-0.1.1}/PKG-INFO +1 -1
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/__init__.py +1 -1
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/versions.py +2 -1
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/reporting/final_view.py +30 -11
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/reporting/run_view.py +4 -1
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/compare.py +19 -4
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/evaluator.py +14 -2
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/final_test.py +2 -2
- strop_cli-0.1.1/tests/integration/test_failures.py +115 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/test_final_and_compare.py +1 -1
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/test_litellm_offline.py +8 -4
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/test_optimize_flow.py +1 -0
- strop_cli-0.1.1/tests/unit/reporting/test_final_view.py +45 -0
- strop_cli-0.1.1/tests/unit/storage/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/test_docs.py +5 -0
- strop_cli-0.1.1/tests/unit/test_formatting.py +15 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/.gitignore +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/LICENSE +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/README.md +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/pyproject.toml +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/__main__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/analysis/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/analysis/classify.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/analysis/evidence.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/analysis/facts.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/analysis/patterns.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/analyze.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/app.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/data.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/evaluate.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/export.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/final.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/init.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/optimize.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/options.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/progress.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/prompts.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/status.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/base.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/constraints.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/dataset.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/enums.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/io.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/limits.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/llm.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/metrics.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/optimization.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/output.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/overrides.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/root.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/rules.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/split.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/data/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/data/loaders.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/data/models.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/data/rows.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/data/splitting.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/env.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/errors.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/evaluation/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/evaluation/budget.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/evaluation/budgeted.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/evaluation/examples.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/evaluation/models.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/evaluation/parsing.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/evaluation/runner.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/formatting.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/caching.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/client.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/factory.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/litellm_client.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/mock.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/retry.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/structured.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/types.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/log.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/basic.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/classification.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/judge.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/python_metric.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/python_runner.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/registry.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/rules.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/structured.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/suite.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/text.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/types.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/optimization/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/optimization/acceptance.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/optimization/generator.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/optimization/meta_prompt.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/optimization/types.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/prompts/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/prompts/diff.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/prompts/files.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/prompts/models.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/prompts/render.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/prompts/template.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/prompts/validation.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/prompts/versions.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/py.typed +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/reporting/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/reporting/console.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/reporting/diff_view.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/reporting/evaluation_view.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/reporting/prompt_export.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/reporting/report_markdown.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/analysis.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/clean.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/context.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/data.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/evaluate.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/export.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/metrics.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/optimize.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/partitions.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/project.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/prompts.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/report.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/review.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/scaffold.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/turn.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/cache.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/database.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/datasets.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/evaluations.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrations/001_initial.sql +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrations/002_datasets.sql +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrations/003_prompt_versions.sql +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrations/004_llm_cache.sql +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrations/005_evaluations.sql +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrations/006_runs.sql +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrations/007_example_feedback.sql +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrations/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrator.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/paths.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/prompts.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/run_models.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/runs.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/conftest.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/helpers.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/conftest.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/test_examples.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/test_export_and_clean.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/test_json_task.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/test_model_overrides.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/analysis/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/analysis/test_facts.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/analysis/test_patterns.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/cli/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/cli/test_data.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/cli/test_evaluate.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/cli/test_init.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/cli/test_init_metrics.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/cli/test_prompts.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/config/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/config/test_constraints.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/config/test_io.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/config/test_rules.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/config/test_sections.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/data/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/data/test_load_dataset.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/data/test_loaders.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/data/test_rows.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/data/test_splitting.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/evaluation/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/evaluation/test_budget.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/evaluation/test_parsing.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/evaluation/test_runner.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/llm/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/llm/test_litellm_client.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/llm/test_mock.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/llm/test_retry_and_cache.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/llm/test_types.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/metrics/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/metrics/test_classification.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/metrics/test_judge.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/metrics/test_python_metric.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/metrics/test_structured.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/metrics/test_suite.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/metrics/test_text_and_rules.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/optimization/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/optimization/test_acceptance.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/optimization/test_generator.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/optimization/test_isolation.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/prompts/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/prompts/test_diff.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/prompts/test_export_formats.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/prompts/test_files.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/prompts/test_template.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/prompts/test_validation.py +0 -0
- {strop_cli-0.1.0/tests/unit/services → strop_cli-0.1.1/tests/unit/reporting}/__init__.py +0 -0
- {strop_cli-0.1.0/tests/unit/storage → strop_cli-0.1.1/tests/unit/services}/__init__.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/services/test_evaluator.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/services/test_metric_setup.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/storage/test_database.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/storage/test_datasets.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/storage/test_evaluations.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/storage/test_migrator.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/storage/test_prompts.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/storage/test_runs.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/test_env.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/test_errors.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/test_log.py +0 -0
- {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/test_package.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: strop-cli
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: Command-line tool that improves LLM prompts by testing them against your own data.
|
|
5
5
|
Project-URL: Homepage, https://github.com/baranylcn/strop
|
|
6
6
|
Project-URL: Documentation, https://github.com/baranylcn/strop/tree/main/docs
|
|
@@ -7,6 +7,7 @@ import typer
|
|
|
7
7
|
|
|
8
8
|
from strop.cli.options import DirOption, JsonOption, emit_json
|
|
9
9
|
from strop.errors import ConfigError
|
|
10
|
+
from strop.formatting import shorten
|
|
10
11
|
from strop.prompts.diff import diff_templates
|
|
11
12
|
from strop.prompts.versions import PromptVersion
|
|
12
13
|
from strop.reporting import ColorMode, make_console, render_diff
|
|
@@ -71,7 +72,7 @@ def history_command(
|
|
|
71
72
|
if decision and decision.primary_delta is not None
|
|
72
73
|
else "-"
|
|
73
74
|
)
|
|
74
|
-
summary = version.change_summary
|
|
75
|
+
summary = shorten(version.change_summary, SUMMARY_WIDTH)
|
|
75
76
|
typer.echo(
|
|
76
77
|
f"{marker} {version.ref:<5} {version.source:<10} parent {parent:<5} "
|
|
77
78
|
f"{status:<9} {delta:<8} {summary}"
|
|
@@ -35,7 +35,7 @@ def render_final_test(console: Console, result: FinalTest) -> None:
|
|
|
35
35
|
table.add_row(result.version.ref, "test", _value(record), str(record.sample_count))
|
|
36
36
|
console.print(table)
|
|
37
37
|
console.print(Text(usage_line(record.usage), style="dim"), soft_wrap=True)
|
|
38
|
-
note =
|
|
38
|
+
note = uncertainty_note(record)
|
|
39
39
|
if note:
|
|
40
40
|
console.print(note, soft_wrap=True, highlight=False)
|
|
41
41
|
console.print(
|
|
@@ -45,16 +45,24 @@ def render_final_test(console: Console, result: FinalTest) -> None:
|
|
|
45
45
|
)
|
|
46
46
|
|
|
47
47
|
|
|
48
|
-
def
|
|
49
|
-
"""
|
|
48
|
+
def wilson_interval(rate: float, n: int, z: float = 1.96) -> tuple[float, float]:
|
|
49
|
+
"""The 95% Wilson score interval for a rate measured on ``n`` examples."""
|
|
50
|
+
denominator = 1 + z * z / n
|
|
51
|
+
center = (rate + z * z / (2 * n)) / denominator
|
|
52
|
+
half = z * math.sqrt(rate * (1 - rate) / n + z * z / (4 * n * n)) / denominator
|
|
53
|
+
return max(0.0, center - half), min(1.0, center + half)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def uncertainty_note(record: EvaluationRecord) -> str | None:
|
|
57
|
+
"""How far a rate-like score could be from the truth, so small test sets are not over-read."""
|
|
50
58
|
value = record.report.primary_value.value
|
|
51
59
|
n = record.report.primary_value.samples
|
|
52
60
|
if value is None or n == 0 or not 0.0 <= value <= 1.0:
|
|
53
61
|
return None
|
|
54
|
-
|
|
62
|
+
low, high = wilson_interval(value, n)
|
|
55
63
|
return (
|
|
56
|
-
f"Note:
|
|
57
|
-
f"
|
|
64
|
+
f"Note: read as a rate over {n} test examples, a score of {value:.2f} is consistent with "
|
|
65
|
+
f"a true value between about {low:.2f} and {high:.2f} (95% Wilson interval)."
|
|
58
66
|
)
|
|
59
67
|
|
|
60
68
|
|
|
@@ -62,17 +70,22 @@ def render_comparison(console: Console, comparison: Comparison) -> None:
|
|
|
62
70
|
title = f"{comparison.version.ref} on {comparison.partition.value}, same rows for every model"
|
|
63
71
|
console.print(Text(title, style="bold"), soft_wrap=True)
|
|
64
72
|
table = Table(box=None, pad_edge=False, header_style="bold")
|
|
65
|
-
|
|
73
|
+
done = [r.record for r in comparison.results if r.record is not None]
|
|
74
|
+
primary = done[0].report.primary
|
|
66
75
|
for column in ("Model", primary, "Valid output rate", "Errors", "Cost", "Mean latency"):
|
|
67
|
-
table.add_column(column)
|
|
68
|
-
for
|
|
76
|
+
table.add_column(column, overflow="fold")
|
|
77
|
+
for result in comparison.results:
|
|
78
|
+
record = result.record
|
|
79
|
+
if record is None:
|
|
80
|
+
table.add_row(result.model, "failed", "-", "-", "-", "-")
|
|
81
|
+
continue
|
|
69
82
|
validity = record.report.values.get("valid_output_rate")
|
|
70
83
|
cost = "unavailable" if record.usage.cost_usd is None else f"${record.usage.cost_usd:.4f}"
|
|
71
84
|
latency = (
|
|
72
85
|
fmt(record.usage.mean_latency_s, 2) + "s" if record.usage.mean_latency_s else "n/a"
|
|
73
86
|
)
|
|
74
87
|
table.add_row(
|
|
75
|
-
|
|
88
|
+
result.model,
|
|
76
89
|
_value(record),
|
|
77
90
|
fmt(validity.value if validity else None),
|
|
78
91
|
f"{record.error_count}/{record.sample_count}",
|
|
@@ -80,6 +93,9 @@ def render_comparison(console: Console, comparison: Comparison) -> None:
|
|
|
80
93
|
latency,
|
|
81
94
|
)
|
|
82
95
|
console.print(table)
|
|
96
|
+
for result in comparison.results:
|
|
97
|
+
if result.error:
|
|
98
|
+
console.print(f"{result.model}: {result.error}", soft_wrap=True, markup=False)
|
|
83
99
|
|
|
84
100
|
|
|
85
101
|
def _value(record: EvaluationRecord) -> str:
|
|
@@ -104,6 +120,9 @@ def comparison_payload(comparison: Comparison) -> dict[str, Any]:
|
|
|
104
120
|
"version": ref,
|
|
105
121
|
"partition": comparison.partition.value,
|
|
106
122
|
"models": [
|
|
107
|
-
{"model":
|
|
123
|
+
{"model": r.model, **record_payload(r.record, ref)}
|
|
124
|
+
if r.record is not None
|
|
125
|
+
else {"model": r.model, "error": r.error}
|
|
126
|
+
for r in comparison.results
|
|
108
127
|
],
|
|
109
128
|
}
|
|
@@ -55,6 +55,8 @@ def render_run(console: Console, summary: RunSummary) -> None:
|
|
|
55
55
|
|
|
56
56
|
|
|
57
57
|
def _render_final_diff(console: Console, summary: RunSummary) -> None:
|
|
58
|
+
if summary.pending is not None:
|
|
59
|
+
return
|
|
58
60
|
if summary.final.number == summary.initial.number:
|
|
59
61
|
console.print("No improvement was accepted; the best prompt is unchanged.", soft_wrap=True)
|
|
60
62
|
return
|
|
@@ -73,7 +75,8 @@ def _candidate_table(candidates: list[CandidateRecord]) -> Table:
|
|
|
73
75
|
for item in candidates:
|
|
74
76
|
delta = "-" if item.primary_delta is None else f"{item.primary_delta:+.4f}"
|
|
75
77
|
version = f"v{item.version_number}" if item.version_number else "-"
|
|
76
|
-
|
|
78
|
+
explained = item.status in {"invalid", "rejected"}
|
|
79
|
+
note = (item.reason or item.change_summary) if explained else item.change_summary
|
|
77
80
|
table.add_row(str(item.turn), version, item.status, delta, shorten(note, 60))
|
|
78
81
|
return table
|
|
79
82
|
|
|
@@ -4,7 +4,7 @@ import sqlite3
|
|
|
4
4
|
from dataclasses import dataclass, replace
|
|
5
5
|
|
|
6
6
|
from strop.data.models import Partition
|
|
7
|
-
from strop.errors import ConfigError
|
|
7
|
+
from strop.errors import ConfigError, LLMError
|
|
8
8
|
from strop.evaluation.models import EvaluationRecord
|
|
9
9
|
from strop.evaluation.runner import Progress
|
|
10
10
|
from strop.prompts.versions import PromptVersion
|
|
@@ -15,11 +15,20 @@ from strop.services.prompts import ensure_initial_version, resolve_version
|
|
|
15
15
|
from strop.storage.prompts import PromptRepository
|
|
16
16
|
|
|
17
17
|
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class ModelResult:
|
|
20
|
+
"""One model's evaluation, or the reason it could not be evaluated."""
|
|
21
|
+
|
|
22
|
+
model: str
|
|
23
|
+
record: EvaluationRecord | None = None
|
|
24
|
+
error: str | None = None
|
|
25
|
+
|
|
26
|
+
|
|
18
27
|
@dataclass(frozen=True)
|
|
19
28
|
class Comparison:
|
|
20
29
|
version: PromptVersion
|
|
21
30
|
partition: Partition
|
|
22
|
-
results: list[
|
|
31
|
+
results: list[ModelResult]
|
|
23
32
|
|
|
24
33
|
|
|
25
34
|
def parse_models(text: str) -> list[str]:
|
|
@@ -55,6 +64,12 @@ async def run_compare( # noqa: PLR0913
|
|
|
55
64
|
for name in models:
|
|
56
65
|
evaluator = replace(base, target=base.target.model_copy(update={"model": name}))
|
|
57
66
|
tag = RunTag("compare", {"sampled": sample is not None})
|
|
58
|
-
|
|
59
|
-
|
|
67
|
+
try:
|
|
68
|
+
record = await evaluator.evaluate(version, partition, tag, rows=rows, progress=progress)
|
|
69
|
+
except LLMError as exc:
|
|
70
|
+
results.append(ModelResult(name, error=str(exc)))
|
|
71
|
+
else:
|
|
72
|
+
results.append(ModelResult(name, record))
|
|
73
|
+
if all(r.record is None for r in results):
|
|
74
|
+
raise LLMError(f"no model could be evaluated; first error: {results[0].error}")
|
|
60
75
|
return Comparison(version, partition, results)
|
|
@@ -11,17 +11,18 @@ from typing import Any
|
|
|
11
11
|
from strop.config.enums import ModelRole
|
|
12
12
|
from strop.config.llm import ModelSpec
|
|
13
13
|
from strop.data.models import Partition, Row
|
|
14
|
-
from strop.errors import DataError
|
|
14
|
+
from strop.errors import DataError, LLMError
|
|
15
15
|
from strop.evaluation.budget import BudgetTracker
|
|
16
16
|
from strop.evaluation.budgeted import BudgetedClient
|
|
17
17
|
from strop.evaluation.examples import to_examples, with_scores
|
|
18
|
-
from strop.evaluation.models import EvaluationRecord, NewEvaluation
|
|
18
|
+
from strop.evaluation.models import EvaluationRecord, ExampleOutcome, NewEvaluation
|
|
19
19
|
from strop.evaluation.parsing import load_output_schema
|
|
20
20
|
from strop.evaluation.runner import Progress, PromptRunner
|
|
21
21
|
from strop.llm.client import LLMClient
|
|
22
22
|
from strop.llm.factory import build_client
|
|
23
23
|
from strop.log import LOGGER_NAME
|
|
24
24
|
from strop.metrics.suite import MetricSuite
|
|
25
|
+
from strop.metrics.types import ErrorCode
|
|
25
26
|
from strop.prompts.versions import PromptVersion
|
|
26
27
|
from strop.services.metrics import build_suite, judge_model
|
|
27
28
|
from strop.services.partitions import PartitionedData, load_partitions
|
|
@@ -113,6 +114,7 @@ class Evaluator:
|
|
|
113
114
|
self.budget,
|
|
114
115
|
)
|
|
115
116
|
outcomes = await runner.run(version.template, selected, progress)
|
|
117
|
+
_require_some_answers(outcomes)
|
|
116
118
|
examples = to_examples(selected, outcomes)
|
|
117
119
|
await self.suite.prepare(examples)
|
|
118
120
|
report = self.suite.evaluate(examples)
|
|
@@ -134,6 +136,16 @@ class Evaluator:
|
|
|
134
136
|
)
|
|
135
137
|
|
|
136
138
|
|
|
139
|
+
def _require_some_answers(outcomes: Sequence[ExampleOutcome]) -> None:
|
|
140
|
+
"""An evaluation where every call failed measures nothing, so it is an error, not a score."""
|
|
141
|
+
if outcomes and all(o.error == ErrorCode.PROVIDER_ERROR for o in outcomes):
|
|
142
|
+
raise LLMError(
|
|
143
|
+
f"every model call failed ({len(outcomes)} of {len(outcomes)}); first error: "
|
|
144
|
+
f"{(outcomes[0].detail or 'unknown').rstrip('.')}. Check the model name, the API key, "
|
|
145
|
+
"the API base, and your network connection."
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
|
|
137
149
|
def build_evaluator(
|
|
138
150
|
project: Project,
|
|
139
151
|
conn: sqlite3.Connection,
|
|
@@ -54,8 +54,8 @@ async def run_final_test( # noqa: PLR0913
|
|
|
54
54
|
]
|
|
55
55
|
if used and not allow_rerun:
|
|
56
56
|
raise ConfigError(
|
|
57
|
-
f"the test partition was already used (evaluation {used[
|
|
58
|
-
f"v{used[
|
|
57
|
+
f"the test partition was already used (evaluation {used[-1].id}, "
|
|
58
|
+
f"v{used[-1].prompt_number}). Testing again after changing the prompt biases the "
|
|
59
59
|
"result; pass --allow-rerun to continue anyway"
|
|
60
60
|
)
|
|
61
61
|
record = await evaluator.evaluate(version, Partition.TEST, RunTag("test"), progress=progress)
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""What happens when the model provider fails."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import sqlite3
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
from strop.errors import LLMError
|
|
10
|
+
from strop.llm import LLMRequest, MockLLMClient
|
|
11
|
+
from tests.helpers import CliRunner
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def install(monkeypatch: pytest.MonkeyPatch, client: MockLLMClient) -> None:
|
|
15
|
+
for module in ("evaluator", "context", "evaluate", "analysis"):
|
|
16
|
+
monkeypatch.setattr(
|
|
17
|
+
f"strop.services.{module}.build_client", lambda *_: client, raising=False
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def always_down(request: LLMRequest) -> str:
|
|
22
|
+
raise LLMError("APIConnectionError: connection refused")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def test_a_run_where_every_call_fails_is_an_error_and_stores_nothing(
|
|
26
|
+
run_cli: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch
|
|
27
|
+
) -> None:
|
|
28
|
+
install(monkeypatch, MockLLMClient(always_down))
|
|
29
|
+
assert run_cli("data", "split", "--dir", str(project))[0] == 0
|
|
30
|
+
code, out, err = run_cli("baseline", "--dir", str(project))
|
|
31
|
+
assert code == 1
|
|
32
|
+
assert "every model call failed (20 of 20)" in err
|
|
33
|
+
assert "connection refused" in err
|
|
34
|
+
assert "Output problems" not in out
|
|
35
|
+
conn = sqlite3.connect(project / ".strop" / "strop.db")
|
|
36
|
+
assert conn.execute("SELECT COUNT(*) FROM evaluations").fetchone()[0] == 0
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_optimize_fails_clearly_and_marks_the_run_failed(
|
|
40
|
+
run_cli: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch
|
|
41
|
+
) -> None:
|
|
42
|
+
install(monkeypatch, MockLLMClient(always_down))
|
|
43
|
+
assert run_cli("data", "split", "--dir", str(project))[0] == 0
|
|
44
|
+
code, _, err = run_cli("optimize", "--dir", str(project))
|
|
45
|
+
assert code == 1
|
|
46
|
+
assert "every model call failed" in err
|
|
47
|
+
conn = sqlite3.connect(project / ".strop" / "strop.db")
|
|
48
|
+
assert conn.execute("SELECT status FROM runs").fetchall() == [("failed",)]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_some_failed_calls_are_recorded_and_the_run_continues(
|
|
52
|
+
run_cli: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch
|
|
53
|
+
) -> None:
|
|
54
|
+
calls = {"n": 0}
|
|
55
|
+
|
|
56
|
+
def flaky(request: LLMRequest) -> str:
|
|
57
|
+
calls["n"] += 1
|
|
58
|
+
if calls["n"] % 4 == 0:
|
|
59
|
+
raise LLMError("temporary outage")
|
|
60
|
+
return "refund"
|
|
61
|
+
|
|
62
|
+
install(monkeypatch, MockLLMClient(flaky))
|
|
63
|
+
assert run_cli("data", "split", "--dir", str(project))[0] == 0
|
|
64
|
+
code, out, _ = run_cli("baseline", "--dir", str(project))
|
|
65
|
+
assert code == 0
|
|
66
|
+
assert "provider_error 5" in out
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def test_compare_does_not_truncate_long_model_names(
|
|
70
|
+
run_cli: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch
|
|
71
|
+
) -> None:
|
|
72
|
+
install(monkeypatch, MockLLMClient("refund"))
|
|
73
|
+
assert run_cli("data", "split", "--dir", str(project))[0] == 0
|
|
74
|
+
long_name = "provider/a-model-with-a-rather-long-name-for-this-test"
|
|
75
|
+
code, out, _ = run_cli(
|
|
76
|
+
"compare", "--dir", str(project), "--models", f"{long_name},provider/short"
|
|
77
|
+
)
|
|
78
|
+
assert code == 0
|
|
79
|
+
assert chr(0x2026) not in out
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def test_compare_keeps_working_models_when_one_model_fails(
|
|
83
|
+
run_cli: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch
|
|
84
|
+
) -> None:
|
|
85
|
+
def route(request: LLMRequest) -> str:
|
|
86
|
+
if request.model == "provider/broken":
|
|
87
|
+
raise LLMError("no access to this model")
|
|
88
|
+
return "refund"
|
|
89
|
+
|
|
90
|
+
install(monkeypatch, MockLLMClient(route))
|
|
91
|
+
assert run_cli("data", "split", "--dir", str(project))[0] == 0
|
|
92
|
+
code, out, _ = run_cli(
|
|
93
|
+
"compare", "--dir", str(project), "--models", "provider/good,provider/broken"
|
|
94
|
+
)
|
|
95
|
+
assert code == 0
|
|
96
|
+
assert "provider/good" in out
|
|
97
|
+
assert "provider/broken" in out
|
|
98
|
+
assert "failed" in out
|
|
99
|
+
assert "no access to this model" in out
|
|
100
|
+
_, raw, _ = run_cli(
|
|
101
|
+
"compare", "--dir", str(project), "--models", "provider/good,provider/broken", "--json"
|
|
102
|
+
)
|
|
103
|
+
models = {m["model"]: m for m in json.loads(raw)["models"]}
|
|
104
|
+
assert "error" in models["provider/broken"]
|
|
105
|
+
assert models["provider/good"]["partition"] == "validation"
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def test_compare_fails_when_no_model_works(
|
|
109
|
+
run_cli: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch
|
|
110
|
+
) -> None:
|
|
111
|
+
install(monkeypatch, MockLLMClient(always_down))
|
|
112
|
+
assert run_cli("data", "split", "--dir", str(project))[0] == 0
|
|
113
|
+
code, _, err = run_cli("compare", "--dir", str(project), "--models", "a/x,b/y")
|
|
114
|
+
assert code == 1
|
|
115
|
+
assert "no model could be evaluated" in err
|
|
@@ -32,7 +32,7 @@ def test_confirmed_test_reports_the_held_out_result(
|
|
|
32
32
|
assert "Held-out test result" in out
|
|
33
33
|
assert "v1 (initial)" in out
|
|
34
34
|
assert "validation" in out
|
|
35
|
-
assert "
|
|
35
|
+
assert "95% Wilson interval" in out
|
|
36
36
|
assert "Do not change the prompt based on this result" in out
|
|
37
37
|
|
|
38
38
|
|
|
@@ -32,12 +32,16 @@ def test_real_client_records_usage_and_serves_repeats_from_the_cache(
|
|
|
32
32
|
assert "Calls: 20 (20 cached)" in again
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
def
|
|
35
|
+
def test_an_unknown_provider_is_reported_instead_of_scored_as_zero(
|
|
36
|
+
run_cli: CliRunner, project: Path
|
|
37
|
+
) -> None:
|
|
36
38
|
path = project / "strop.yaml"
|
|
37
39
|
path.write_text(
|
|
38
40
|
path.read_text("utf-8").replace("openai/model-a", "no-such-provider/x"), "utf-8"
|
|
39
41
|
)
|
|
40
42
|
assert run_cli("data", "split", "--dir", str(project))[0] == 0
|
|
41
|
-
code, out,
|
|
42
|
-
assert code ==
|
|
43
|
-
assert "
|
|
43
|
+
code, out, err = run_cli("baseline", "--dir", str(project))
|
|
44
|
+
assert code == 1
|
|
45
|
+
assert "every model call failed (20 of 20)" in err
|
|
46
|
+
assert "Check the model name, the API key" in err
|
|
47
|
+
assert "macro_f1" not in out
|
|
@@ -57,6 +57,7 @@ def test_manual_approval_pauses_then_resumes(
|
|
|
57
57
|
code, out, err = optimize(run_cli, ready, "--approval", "manual")
|
|
58
58
|
assert code == 10
|
|
59
59
|
assert "paused for approval" in out
|
|
60
|
+
assert "No improvement was accepted" not in out
|
|
60
61
|
assert "v2 is waiting for 'strop approve' or 'strop reject'" in out
|
|
61
62
|
assert "strop approve v2" in err
|
|
62
63
|
assert history(run_cli, ready)[0].startswith("* v1")
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import pytest
|
|
2
|
+
|
|
3
|
+
from strop.data.models import Partition
|
|
4
|
+
from strop.evaluation.models import EvaluationRecord, UsageSummary
|
|
5
|
+
from strop.metrics.suite import MetricReport
|
|
6
|
+
from strop.metrics.types import MetricValue
|
|
7
|
+
from strop.reporting.final_view import uncertainty_note, wilson_interval
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def record(value: float | None, samples: int) -> EvaluationRecord:
|
|
11
|
+
report = MetricReport("m", {"m": MetricValue("m", value, samples)})
|
|
12
|
+
return EvaluationRecord(1, 1, 1, Partition.TEST, "test", report, UsageSummary(), samples, 0, "")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_wilson_interval_for_a_perfect_score_is_not_zero_width() -> None:
|
|
16
|
+
low, high = wilson_interval(1.0, 30)
|
|
17
|
+
assert low == pytest.approx(0.886, abs=0.005)
|
|
18
|
+
assert high == 1.0
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_wilson_interval_matches_a_known_value() -> None:
|
|
22
|
+
low, high = wilson_interval(0.5, 100)
|
|
23
|
+
assert (round(low, 3), round(high, 3)) == (0.404, 0.596)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_interval_narrows_as_the_sample_grows() -> None:
|
|
27
|
+
small = wilson_interval(0.8, 30)
|
|
28
|
+
large = wilson_interval(0.8, 3000)
|
|
29
|
+
assert (large[1] - large[0]) < (small[1] - small[0]) / 5
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_note_describes_the_interval() -> None:
|
|
33
|
+
note = uncertainty_note(record(1.0, 30))
|
|
34
|
+
assert note is not None
|
|
35
|
+
assert "30 test examples" in note
|
|
36
|
+
assert "between about 0.89 and 1.00" in note
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@pytest.mark.parametrize("value", [None, -0.1, 1.5])
|
|
40
|
+
def test_no_note_when_the_score_is_not_a_rate(value: float | None) -> None:
|
|
41
|
+
assert uncertainty_note(record(value, 30)) is None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_no_note_without_samples() -> None:
|
|
45
|
+
assert uncertainty_note(record(0.5, 0)) is None
|
|
File without changes
|
|
@@ -80,3 +80,8 @@ def test_every_configuration_key_is_documented() -> None:
|
|
|
80
80
|
if key in {"model", "temperature", "max_tokens", "timeout_s", "params"}:
|
|
81
81
|
continue
|
|
82
82
|
assert f"`{key}`" in text, f"{section}.{key}"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_the_changelog_has_an_entry_for_the_current_version() -> None:
|
|
86
|
+
text = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8")
|
|
87
|
+
assert f"## [{strop.__version__}]" in text
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from strop.formatting import fmt, shorten
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def test_fmt_uses_fixed_decimals_and_marks_missing_values() -> None:
|
|
5
|
+
assert fmt(0.5) == "0.5000"
|
|
6
|
+
assert fmt(0.123456, 2) == "0.12"
|
|
7
|
+
assert fmt(None) == "n/a"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_shorten_collapses_whitespace_and_marks_the_cut() -> None:
|
|
11
|
+
assert shorten("a\n b\tc") == "a b c"
|
|
12
|
+
cut = shorten("word " * 30, 20)
|
|
13
|
+
assert len(cut) == 20
|
|
14
|
+
assert cut.endswith("...")
|
|
15
|
+
assert shorten("short", 20) == "short"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|