strop-cli 0.1.0__tar.gz → 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (214) hide show
  1. {strop_cli-0.1.0 → strop_cli-0.1.1}/PKG-INFO +1 -1
  2. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/__init__.py +1 -1
  3. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/versions.py +2 -1
  4. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/reporting/final_view.py +30 -11
  5. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/reporting/run_view.py +4 -1
  6. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/compare.py +19 -4
  7. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/evaluator.py +14 -2
  8. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/final_test.py +2 -2
  9. strop_cli-0.1.1/tests/integration/test_failures.py +115 -0
  10. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/test_final_and_compare.py +1 -1
  11. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/test_litellm_offline.py +8 -4
  12. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/test_optimize_flow.py +1 -0
  13. strop_cli-0.1.1/tests/unit/reporting/test_final_view.py +45 -0
  14. strop_cli-0.1.1/tests/unit/storage/__init__.py +0 -0
  15. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/test_docs.py +5 -0
  16. strop_cli-0.1.1/tests/unit/test_formatting.py +15 -0
  17. {strop_cli-0.1.0 → strop_cli-0.1.1}/.gitignore +0 -0
  18. {strop_cli-0.1.0 → strop_cli-0.1.1}/LICENSE +0 -0
  19. {strop_cli-0.1.0 → strop_cli-0.1.1}/README.md +0 -0
  20. {strop_cli-0.1.0 → strop_cli-0.1.1}/pyproject.toml +0 -0
  21. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/__main__.py +0 -0
  22. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/analysis/__init__.py +0 -0
  23. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/analysis/classify.py +0 -0
  24. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/analysis/evidence.py +0 -0
  25. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/analysis/facts.py +0 -0
  26. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/analysis/patterns.py +0 -0
  27. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/__init__.py +0 -0
  28. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/analyze.py +0 -0
  29. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/app.py +0 -0
  30. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/data.py +0 -0
  31. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/evaluate.py +0 -0
  32. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/export.py +0 -0
  33. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/final.py +0 -0
  34. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/init.py +0 -0
  35. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/optimize.py +0 -0
  36. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/options.py +0 -0
  37. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/progress.py +0 -0
  38. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/prompts.py +0 -0
  39. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/cli/status.py +0 -0
  40. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/__init__.py +0 -0
  41. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/base.py +0 -0
  42. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/constraints.py +0 -0
  43. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/dataset.py +0 -0
  44. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/enums.py +0 -0
  45. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/io.py +0 -0
  46. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/limits.py +0 -0
  47. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/llm.py +0 -0
  48. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/metrics.py +0 -0
  49. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/optimization.py +0 -0
  50. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/output.py +0 -0
  51. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/overrides.py +0 -0
  52. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/root.py +0 -0
  53. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/rules.py +0 -0
  54. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/config/split.py +0 -0
  55. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/data/__init__.py +0 -0
  56. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/data/loaders.py +0 -0
  57. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/data/models.py +0 -0
  58. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/data/rows.py +0 -0
  59. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/data/splitting.py +0 -0
  60. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/env.py +0 -0
  61. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/errors.py +0 -0
  62. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/evaluation/__init__.py +0 -0
  63. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/evaluation/budget.py +0 -0
  64. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/evaluation/budgeted.py +0 -0
  65. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/evaluation/examples.py +0 -0
  66. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/evaluation/models.py +0 -0
  67. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/evaluation/parsing.py +0 -0
  68. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/evaluation/runner.py +0 -0
  69. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/formatting.py +0 -0
  70. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/__init__.py +0 -0
  71. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/caching.py +0 -0
  72. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/client.py +0 -0
  73. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/factory.py +0 -0
  74. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/litellm_client.py +0 -0
  75. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/mock.py +0 -0
  76. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/retry.py +0 -0
  77. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/structured.py +0 -0
  78. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/llm/types.py +0 -0
  79. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/log.py +0 -0
  80. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/__init__.py +0 -0
  81. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/basic.py +0 -0
  82. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/classification.py +0 -0
  83. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/judge.py +0 -0
  84. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/python_metric.py +0 -0
  85. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/python_runner.py +0 -0
  86. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/registry.py +0 -0
  87. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/rules.py +0 -0
  88. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/structured.py +0 -0
  89. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/suite.py +0 -0
  90. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/text.py +0 -0
  91. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/metrics/types.py +0 -0
  92. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/optimization/__init__.py +0 -0
  93. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/optimization/acceptance.py +0 -0
  94. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/optimization/generator.py +0 -0
  95. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/optimization/meta_prompt.py +0 -0
  96. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/optimization/types.py +0 -0
  97. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/prompts/__init__.py +0 -0
  98. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/prompts/diff.py +0 -0
  99. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/prompts/files.py +0 -0
  100. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/prompts/models.py +0 -0
  101. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/prompts/render.py +0 -0
  102. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/prompts/template.py +0 -0
  103. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/prompts/validation.py +0 -0
  104. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/prompts/versions.py +0 -0
  105. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/py.typed +0 -0
  106. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/reporting/__init__.py +0 -0
  107. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/reporting/console.py +0 -0
  108. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/reporting/diff_view.py +0 -0
  109. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/reporting/evaluation_view.py +0 -0
  110. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/reporting/prompt_export.py +0 -0
  111. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/reporting/report_markdown.py +0 -0
  112. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/__init__.py +0 -0
  113. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/analysis.py +0 -0
  114. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/clean.py +0 -0
  115. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/context.py +0 -0
  116. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/data.py +0 -0
  117. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/evaluate.py +0 -0
  118. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/export.py +0 -0
  119. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/metrics.py +0 -0
  120. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/optimize.py +0 -0
  121. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/partitions.py +0 -0
  122. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/project.py +0 -0
  123. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/prompts.py +0 -0
  124. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/report.py +0 -0
  125. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/review.py +0 -0
  126. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/scaffold.py +0 -0
  127. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/services/turn.py +0 -0
  128. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/__init__.py +0 -0
  129. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/cache.py +0 -0
  130. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/database.py +0 -0
  131. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/datasets.py +0 -0
  132. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/evaluations.py +0 -0
  133. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrations/001_initial.sql +0 -0
  134. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrations/002_datasets.sql +0 -0
  135. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrations/003_prompt_versions.sql +0 -0
  136. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrations/004_llm_cache.sql +0 -0
  137. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrations/005_evaluations.sql +0 -0
  138. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrations/006_runs.sql +0 -0
  139. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrations/007_example_feedback.sql +0 -0
  140. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrations/__init__.py +0 -0
  141. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/migrator.py +0 -0
  142. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/paths.py +0 -0
  143. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/prompts.py +0 -0
  144. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/run_models.py +0 -0
  145. {strop_cli-0.1.0 → strop_cli-0.1.1}/src/strop/storage/runs.py +0 -0
  146. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/__init__.py +0 -0
  147. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/conftest.py +0 -0
  148. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/helpers.py +0 -0
  149. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/__init__.py +0 -0
  150. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/conftest.py +0 -0
  151. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/test_examples.py +0 -0
  152. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/test_export_and_clean.py +0 -0
  153. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/test_json_task.py +0 -0
  154. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/integration/test_model_overrides.py +0 -0
  155. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/__init__.py +0 -0
  156. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/analysis/__init__.py +0 -0
  157. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/analysis/test_facts.py +0 -0
  158. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/analysis/test_patterns.py +0 -0
  159. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/cli/__init__.py +0 -0
  160. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/cli/test_data.py +0 -0
  161. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/cli/test_evaluate.py +0 -0
  162. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/cli/test_init.py +0 -0
  163. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/cli/test_init_metrics.py +0 -0
  164. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/cli/test_prompts.py +0 -0
  165. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/config/__init__.py +0 -0
  166. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/config/test_constraints.py +0 -0
  167. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/config/test_io.py +0 -0
  168. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/config/test_rules.py +0 -0
  169. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/config/test_sections.py +0 -0
  170. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/data/__init__.py +0 -0
  171. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/data/test_load_dataset.py +0 -0
  172. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/data/test_loaders.py +0 -0
  173. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/data/test_rows.py +0 -0
  174. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/data/test_splitting.py +0 -0
  175. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/evaluation/__init__.py +0 -0
  176. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/evaluation/test_budget.py +0 -0
  177. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/evaluation/test_parsing.py +0 -0
  178. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/evaluation/test_runner.py +0 -0
  179. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/llm/__init__.py +0 -0
  180. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/llm/test_litellm_client.py +0 -0
  181. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/llm/test_mock.py +0 -0
  182. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/llm/test_retry_and_cache.py +0 -0
  183. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/llm/test_types.py +0 -0
  184. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/metrics/__init__.py +0 -0
  185. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/metrics/test_classification.py +0 -0
  186. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/metrics/test_judge.py +0 -0
  187. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/metrics/test_python_metric.py +0 -0
  188. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/metrics/test_structured.py +0 -0
  189. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/metrics/test_suite.py +0 -0
  190. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/metrics/test_text_and_rules.py +0 -0
  191. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/optimization/__init__.py +0 -0
  192. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/optimization/test_acceptance.py +0 -0
  193. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/optimization/test_generator.py +0 -0
  194. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/optimization/test_isolation.py +0 -0
  195. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/prompts/__init__.py +0 -0
  196. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/prompts/test_diff.py +0 -0
  197. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/prompts/test_export_formats.py +0 -0
  198. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/prompts/test_files.py +0 -0
  199. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/prompts/test_template.py +0 -0
  200. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/prompts/test_validation.py +0 -0
  201. {strop_cli-0.1.0/tests/unit/services → strop_cli-0.1.1/tests/unit/reporting}/__init__.py +0 -0
  202. {strop_cli-0.1.0/tests/unit/storage → strop_cli-0.1.1/tests/unit/services}/__init__.py +0 -0
  203. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/services/test_evaluator.py +0 -0
  204. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/services/test_metric_setup.py +0 -0
  205. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/storage/test_database.py +0 -0
  206. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/storage/test_datasets.py +0 -0
  207. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/storage/test_evaluations.py +0 -0
  208. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/storage/test_migrator.py +0 -0
  209. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/storage/test_prompts.py +0 -0
  210. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/storage/test_runs.py +0 -0
  211. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/test_env.py +0 -0
  212. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/test_errors.py +0 -0
  213. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/test_log.py +0 -0
  214. {strop_cli-0.1.0 → strop_cli-0.1.1}/tests/unit/test_package.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: strop-cli
3
- Version: 0.1.0
3
+ Version: 0.1.1
4
4
  Summary: Command-line tool that improves LLM prompts by testing them against your own data.
5
5
  Project-URL: Homepage, https://github.com/baranylcn/strop
6
6
  Project-URL: Documentation, https://github.com/baranylcn/strop/tree/main/docs
@@ -1,5 +1,5 @@
1
1
  """Strop: improve LLM prompts by testing them against your own data."""
2
2
 
3
- __version__ = "0.1.0"
3
+ __version__ = "0.1.1"
4
4
 
5
5
  __all__ = ["__version__"]
@@ -7,6 +7,7 @@ import typer
7
7
 
8
8
  from strop.cli.options import DirOption, JsonOption, emit_json
9
9
  from strop.errors import ConfigError
10
+ from strop.formatting import shorten
10
11
  from strop.prompts.diff import diff_templates
11
12
  from strop.prompts.versions import PromptVersion
12
13
  from strop.reporting import ColorMode, make_console, render_diff
@@ -71,7 +72,7 @@ def history_command(
71
72
  if decision and decision.primary_delta is not None
72
73
  else "-"
73
74
  )
74
- summary = version.change_summary[:SUMMARY_WIDTH]
75
+ summary = shorten(version.change_summary, SUMMARY_WIDTH)
75
76
  typer.echo(
76
77
  f"{marker} {version.ref:<5} {version.source:<10} parent {parent:<5} "
77
78
  f"{status:<9} {delta:<8} {summary}"
@@ -35,7 +35,7 @@ def render_final_test(console: Console, result: FinalTest) -> None:
35
35
  table.add_row(result.version.ref, "test", _value(record), str(record.sample_count))
36
36
  console.print(table)
37
37
  console.print(Text(usage_line(record.usage), style="dim"), soft_wrap=True)
38
- note = standard_error_note(record)
38
+ note = uncertainty_note(record)
39
39
  if note:
40
40
  console.print(note, soft_wrap=True, highlight=False)
41
41
  console.print(
@@ -45,16 +45,24 @@ def render_final_test(console: Console, result: FinalTest) -> None:
45
45
  )
46
46
 
47
47
 
48
- def standard_error_note(record: EvaluationRecord) -> str | None:
49
- """A rough noise estimate for rate-like metrics, so small test sets are not over-read."""
48
+ def wilson_interval(rate: float, n: int, z: float = 1.96) -> tuple[float, float]:
49
+ """The 95% Wilson score interval for a rate measured on ``n`` examples."""
50
+ denominator = 1 + z * z / n
51
+ center = (rate + z * z / (2 * n)) / denominator
52
+ half = z * math.sqrt(rate * (1 - rate) / n + z * z / (4 * n * n)) / denominator
53
+ return max(0.0, center - half), min(1.0, center + half)
54
+
55
+
56
+ def uncertainty_note(record: EvaluationRecord) -> str | None:
57
+ """How far a rate-like score could be from the truth, so small test sets are not over-read."""
50
58
  value = record.report.primary_value.value
51
59
  n = record.report.primary_value.samples
52
60
  if value is None or n == 0 or not 0.0 <= value <= 1.0:
53
61
  return None
54
- error = math.sqrt(value * (1 - value) / n)
62
+ low, high = wilson_interval(value, n)
55
63
  return (
56
- f"Note: with {n} test examples, differences smaller than about {error:.2f} "
57
- f"(one standard error for a rate near {value:.2f}) may be noise."
64
+ f"Note: read as a rate over {n} test examples, a score of {value:.2f} is consistent with "
65
+ f"a true value between about {low:.2f} and {high:.2f} (95% Wilson interval)."
58
66
  )
59
67
 
60
68
 
@@ -62,17 +70,22 @@ def render_comparison(console: Console, comparison: Comparison) -> None:
62
70
  title = f"{comparison.version.ref} on {comparison.partition.value}, same rows for every model"
63
71
  console.print(Text(title, style="bold"), soft_wrap=True)
64
72
  table = Table(box=None, pad_edge=False, header_style="bold")
65
- primary = comparison.results[0][1].report.primary
73
+ done = [r.record for r in comparison.results if r.record is not None]
74
+ primary = done[0].report.primary
66
75
  for column in ("Model", primary, "Valid output rate", "Errors", "Cost", "Mean latency"):
67
- table.add_column(column)
68
- for name, record in comparison.results:
76
+ table.add_column(column, overflow="fold")
77
+ for result in comparison.results:
78
+ record = result.record
79
+ if record is None:
80
+ table.add_row(result.model, "failed", "-", "-", "-", "-")
81
+ continue
69
82
  validity = record.report.values.get("valid_output_rate")
70
83
  cost = "unavailable" if record.usage.cost_usd is None else f"${record.usage.cost_usd:.4f}"
71
84
  latency = (
72
85
  fmt(record.usage.mean_latency_s, 2) + "s" if record.usage.mean_latency_s else "n/a"
73
86
  )
74
87
  table.add_row(
75
- name,
88
+ result.model,
76
89
  _value(record),
77
90
  fmt(validity.value if validity else None),
78
91
  f"{record.error_count}/{record.sample_count}",
@@ -80,6 +93,9 @@ def render_comparison(console: Console, comparison: Comparison) -> None:
80
93
  latency,
81
94
  )
82
95
  console.print(table)
96
+ for result in comparison.results:
97
+ if result.error:
98
+ console.print(f"{result.model}: {result.error}", soft_wrap=True, markup=False)
83
99
 
84
100
 
85
101
  def _value(record: EvaluationRecord) -> str:
@@ -104,6 +120,9 @@ def comparison_payload(comparison: Comparison) -> dict[str, Any]:
104
120
  "version": ref,
105
121
  "partition": comparison.partition.value,
106
122
  "models": [
107
- {"model": name, **record_payload(record, ref)} for name, record in comparison.results
123
+ {"model": r.model, **record_payload(r.record, ref)}
124
+ if r.record is not None
125
+ else {"model": r.model, "error": r.error}
126
+ for r in comparison.results
108
127
  ],
109
128
  }
@@ -55,6 +55,8 @@ def render_run(console: Console, summary: RunSummary) -> None:
55
55
 
56
56
 
57
57
  def _render_final_diff(console: Console, summary: RunSummary) -> None:
58
+ if summary.pending is not None:
59
+ return
58
60
  if summary.final.number == summary.initial.number:
59
61
  console.print("No improvement was accepted; the best prompt is unchanged.", soft_wrap=True)
60
62
  return
@@ -73,7 +75,8 @@ def _candidate_table(candidates: list[CandidateRecord]) -> Table:
73
75
  for item in candidates:
74
76
  delta = "-" if item.primary_delta is None else f"{item.primary_delta:+.4f}"
75
77
  version = f"v{item.version_number}" if item.version_number else "-"
76
- note = item.change_summary or item.reason
78
+ explained = item.status in {"invalid", "rejected"}
79
+ note = (item.reason or item.change_summary) if explained else item.change_summary
77
80
  table.add_row(str(item.turn), version, item.status, delta, shorten(note, 60))
78
81
  return table
79
82
 
@@ -4,7 +4,7 @@ import sqlite3
4
4
  from dataclasses import dataclass, replace
5
5
 
6
6
  from strop.data.models import Partition
7
- from strop.errors import ConfigError
7
+ from strop.errors import ConfigError, LLMError
8
8
  from strop.evaluation.models import EvaluationRecord
9
9
  from strop.evaluation.runner import Progress
10
10
  from strop.prompts.versions import PromptVersion
@@ -15,11 +15,20 @@ from strop.services.prompts import ensure_initial_version, resolve_version
15
15
  from strop.storage.prompts import PromptRepository
16
16
 
17
17
 
18
+ @dataclass(frozen=True)
19
+ class ModelResult:
20
+ """One model's evaluation, or the reason it could not be evaluated."""
21
+
22
+ model: str
23
+ record: EvaluationRecord | None = None
24
+ error: str | None = None
25
+
26
+
18
27
  @dataclass(frozen=True)
19
28
  class Comparison:
20
29
  version: PromptVersion
21
30
  partition: Partition
22
- results: list[tuple[str, EvaluationRecord]]
31
+ results: list[ModelResult]
23
32
 
24
33
 
25
34
  def parse_models(text: str) -> list[str]:
@@ -55,6 +64,12 @@ async def run_compare( # noqa: PLR0913
55
64
  for name in models:
56
65
  evaluator = replace(base, target=base.target.model_copy(update={"model": name}))
57
66
  tag = RunTag("compare", {"sampled": sample is not None})
58
- record = await evaluator.evaluate(version, partition, tag, rows=rows, progress=progress)
59
- results.append((name, record))
67
+ try:
68
+ record = await evaluator.evaluate(version, partition, tag, rows=rows, progress=progress)
69
+ except LLMError as exc:
70
+ results.append(ModelResult(name, error=str(exc)))
71
+ else:
72
+ results.append(ModelResult(name, record))
73
+ if all(r.record is None for r in results):
74
+ raise LLMError(f"no model could be evaluated; first error: {results[0].error}")
60
75
  return Comparison(version, partition, results)
@@ -11,17 +11,18 @@ from typing import Any
11
11
  from strop.config.enums import ModelRole
12
12
  from strop.config.llm import ModelSpec
13
13
  from strop.data.models import Partition, Row
14
- from strop.errors import DataError
14
+ from strop.errors import DataError, LLMError
15
15
  from strop.evaluation.budget import BudgetTracker
16
16
  from strop.evaluation.budgeted import BudgetedClient
17
17
  from strop.evaluation.examples import to_examples, with_scores
18
- from strop.evaluation.models import EvaluationRecord, NewEvaluation
18
+ from strop.evaluation.models import EvaluationRecord, ExampleOutcome, NewEvaluation
19
19
  from strop.evaluation.parsing import load_output_schema
20
20
  from strop.evaluation.runner import Progress, PromptRunner
21
21
  from strop.llm.client import LLMClient
22
22
  from strop.llm.factory import build_client
23
23
  from strop.log import LOGGER_NAME
24
24
  from strop.metrics.suite import MetricSuite
25
+ from strop.metrics.types import ErrorCode
25
26
  from strop.prompts.versions import PromptVersion
26
27
  from strop.services.metrics import build_suite, judge_model
27
28
  from strop.services.partitions import PartitionedData, load_partitions
@@ -113,6 +114,7 @@ class Evaluator:
113
114
  self.budget,
114
115
  )
115
116
  outcomes = await runner.run(version.template, selected, progress)
117
+ _require_some_answers(outcomes)
116
118
  examples = to_examples(selected, outcomes)
117
119
  await self.suite.prepare(examples)
118
120
  report = self.suite.evaluate(examples)
@@ -134,6 +136,16 @@ class Evaluator:
134
136
  )
135
137
 
136
138
 
139
+ def _require_some_answers(outcomes: Sequence[ExampleOutcome]) -> None:
140
+ """An evaluation where every call failed measures nothing, so it is an error, not a score."""
141
+ if outcomes and all(o.error == ErrorCode.PROVIDER_ERROR for o in outcomes):
142
+ raise LLMError(
143
+ f"every model call failed ({len(outcomes)} of {len(outcomes)}); first error: "
144
+ f"{(outcomes[0].detail or 'unknown').rstrip('.')}. Check the model name, the API key, "
145
+ "the API base, and your network connection."
146
+ )
147
+
148
+
137
149
  def build_evaluator(
138
150
  project: Project,
139
151
  conn: sqlite3.Connection,
@@ -54,8 +54,8 @@ async def run_final_test( # noqa: PLR0913
54
54
  ]
55
55
  if used and not allow_rerun:
56
56
  raise ConfigError(
57
- f"the test partition was already used (evaluation {used[0].id}, "
58
- f"v{used[0].prompt_number}). Testing again after changing the prompt biases the "
57
+ f"the test partition was already used (evaluation {used[-1].id}, "
58
+ f"v{used[-1].prompt_number}). Testing again after changing the prompt biases the "
59
59
  "result; pass --allow-rerun to continue anyway"
60
60
  )
61
61
  record = await evaluator.evaluate(version, Partition.TEST, RunTag("test"), progress=progress)
@@ -0,0 +1,115 @@
1
+ """What happens when the model provider fails."""
2
+
3
+ import json
4
+ import sqlite3
5
+ from pathlib import Path
6
+
7
+ import pytest
8
+
9
+ from strop.errors import LLMError
10
+ from strop.llm import LLMRequest, MockLLMClient
11
+ from tests.helpers import CliRunner
12
+
13
+
14
+ def install(monkeypatch: pytest.MonkeyPatch, client: MockLLMClient) -> None:
15
+ for module in ("evaluator", "context", "evaluate", "analysis"):
16
+ monkeypatch.setattr(
17
+ f"strop.services.{module}.build_client", lambda *_: client, raising=False
18
+ )
19
+
20
+
21
+ def always_down(request: LLMRequest) -> str:
22
+ raise LLMError("APIConnectionError: connection refused")
23
+
24
+
25
+ def test_a_run_where_every_call_fails_is_an_error_and_stores_nothing(
26
+ run_cli: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch
27
+ ) -> None:
28
+ install(monkeypatch, MockLLMClient(always_down))
29
+ assert run_cli("data", "split", "--dir", str(project))[0] == 0
30
+ code, out, err = run_cli("baseline", "--dir", str(project))
31
+ assert code == 1
32
+ assert "every model call failed (20 of 20)" in err
33
+ assert "connection refused" in err
34
+ assert "Output problems" not in out
35
+ conn = sqlite3.connect(project / ".strop" / "strop.db")
36
+ assert conn.execute("SELECT COUNT(*) FROM evaluations").fetchone()[0] == 0
37
+
38
+
39
+ def test_optimize_fails_clearly_and_marks_the_run_failed(
40
+ run_cli: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch
41
+ ) -> None:
42
+ install(monkeypatch, MockLLMClient(always_down))
43
+ assert run_cli("data", "split", "--dir", str(project))[0] == 0
44
+ code, _, err = run_cli("optimize", "--dir", str(project))
45
+ assert code == 1
46
+ assert "every model call failed" in err
47
+ conn = sqlite3.connect(project / ".strop" / "strop.db")
48
+ assert conn.execute("SELECT status FROM runs").fetchall() == [("failed",)]
49
+
50
+
51
+ def test_some_failed_calls_are_recorded_and_the_run_continues(
52
+ run_cli: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch
53
+ ) -> None:
54
+ calls = {"n": 0}
55
+
56
+ def flaky(request: LLMRequest) -> str:
57
+ calls["n"] += 1
58
+ if calls["n"] % 4 == 0:
59
+ raise LLMError("temporary outage")
60
+ return "refund"
61
+
62
+ install(monkeypatch, MockLLMClient(flaky))
63
+ assert run_cli("data", "split", "--dir", str(project))[0] == 0
64
+ code, out, _ = run_cli("baseline", "--dir", str(project))
65
+ assert code == 0
66
+ assert "provider_error 5" in out
67
+
68
+
69
+ def test_compare_does_not_truncate_long_model_names(
70
+ run_cli: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch
71
+ ) -> None:
72
+ install(monkeypatch, MockLLMClient("refund"))
73
+ assert run_cli("data", "split", "--dir", str(project))[0] == 0
74
+ long_name = "provider/a-model-with-a-rather-long-name-for-this-test"
75
+ code, out, _ = run_cli(
76
+ "compare", "--dir", str(project), "--models", f"{long_name},provider/short"
77
+ )
78
+ assert code == 0
79
+ assert chr(0x2026) not in out
80
+
81
+
82
+ def test_compare_keeps_working_models_when_one_model_fails(
83
+ run_cli: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch
84
+ ) -> None:
85
+ def route(request: LLMRequest) -> str:
86
+ if request.model == "provider/broken":
87
+ raise LLMError("no access to this model")
88
+ return "refund"
89
+
90
+ install(monkeypatch, MockLLMClient(route))
91
+ assert run_cli("data", "split", "--dir", str(project))[0] == 0
92
+ code, out, _ = run_cli(
93
+ "compare", "--dir", str(project), "--models", "provider/good,provider/broken"
94
+ )
95
+ assert code == 0
96
+ assert "provider/good" in out
97
+ assert "provider/broken" in out
98
+ assert "failed" in out
99
+ assert "no access to this model" in out
100
+ _, raw, _ = run_cli(
101
+ "compare", "--dir", str(project), "--models", "provider/good,provider/broken", "--json"
102
+ )
103
+ models = {m["model"]: m for m in json.loads(raw)["models"]}
104
+ assert "error" in models["provider/broken"]
105
+ assert models["provider/good"]["partition"] == "validation"
106
+
107
+
108
+ def test_compare_fails_when_no_model_works(
109
+ run_cli: CliRunner, project: Path, monkeypatch: pytest.MonkeyPatch
110
+ ) -> None:
111
+ install(monkeypatch, MockLLMClient(always_down))
112
+ assert run_cli("data", "split", "--dir", str(project))[0] == 0
113
+ code, _, err = run_cli("compare", "--dir", str(project), "--models", "a/x,b/y")
114
+ assert code == 1
115
+ assert "no model could be evaluated" in err
@@ -32,7 +32,7 @@ def test_confirmed_test_reports_the_held_out_result(
32
32
  assert "Held-out test result" in out
33
33
  assert "v1 (initial)" in out
34
34
  assert "validation" in out
35
- assert "standard error" in out
35
+ assert "95% Wilson interval" in out
36
36
  assert "Do not change the prompt based on this result" in out
37
37
 
38
38
 
@@ -32,12 +32,16 @@ def test_real_client_records_usage_and_serves_repeats_from_the_cache(
32
32
  assert "Calls: 20 (20 cached)" in again
33
33
 
34
34
 
35
- def test_unknown_providers_fail_without_crashing_the_run(run_cli: CliRunner, project: Path) -> None:
35
+ def test_an_unknown_provider_is_reported_instead_of_scored_as_zero(
36
+ run_cli: CliRunner, project: Path
37
+ ) -> None:
36
38
  path = project / "strop.yaml"
37
39
  path.write_text(
38
40
  path.read_text("utf-8").replace("openai/model-a", "no-such-provider/x"), "utf-8"
39
41
  )
40
42
  assert run_cli("data", "split", "--dir", str(project))[0] == 0
41
- code, out, _ = run_cli("baseline", "--dir", str(project))
42
- assert code == 0
43
- assert "provider_error 20" in out
43
+ code, out, err = run_cli("baseline", "--dir", str(project))
44
+ assert code == 1
45
+ assert "every model call failed (20 of 20)" in err
46
+ assert "Check the model name, the API key" in err
47
+ assert "macro_f1" not in out
@@ -57,6 +57,7 @@ def test_manual_approval_pauses_then_resumes(
57
57
  code, out, err = optimize(run_cli, ready, "--approval", "manual")
58
58
  assert code == 10
59
59
  assert "paused for approval" in out
60
+ assert "No improvement was accepted" not in out
60
61
  assert "v2 is waiting for 'strop approve' or 'strop reject'" in out
61
62
  assert "strop approve v2" in err
62
63
  assert history(run_cli, ready)[0].startswith("* v1")
@@ -0,0 +1,45 @@
1
+ import pytest
2
+
3
+ from strop.data.models import Partition
4
+ from strop.evaluation.models import EvaluationRecord, UsageSummary
5
+ from strop.metrics.suite import MetricReport
6
+ from strop.metrics.types import MetricValue
7
+ from strop.reporting.final_view import uncertainty_note, wilson_interval
8
+
9
+
10
+ def record(value: float | None, samples: int) -> EvaluationRecord:
11
+ report = MetricReport("m", {"m": MetricValue("m", value, samples)})
12
+ return EvaluationRecord(1, 1, 1, Partition.TEST, "test", report, UsageSummary(), samples, 0, "")
13
+
14
+
15
+ def test_wilson_interval_for_a_perfect_score_is_not_zero_width() -> None:
16
+ low, high = wilson_interval(1.0, 30)
17
+ assert low == pytest.approx(0.886, abs=0.005)
18
+ assert high == 1.0
19
+
20
+
21
+ def test_wilson_interval_matches_a_known_value() -> None:
22
+ low, high = wilson_interval(0.5, 100)
23
+ assert (round(low, 3), round(high, 3)) == (0.404, 0.596)
24
+
25
+
26
+ def test_interval_narrows_as_the_sample_grows() -> None:
27
+ small = wilson_interval(0.8, 30)
28
+ large = wilson_interval(0.8, 3000)
29
+ assert (large[1] - large[0]) < (small[1] - small[0]) / 5
30
+
31
+
32
+ def test_note_describes_the_interval() -> None:
33
+ note = uncertainty_note(record(1.0, 30))
34
+ assert note is not None
35
+ assert "30 test examples" in note
36
+ assert "between about 0.89 and 1.00" in note
37
+
38
+
39
+ @pytest.mark.parametrize("value", [None, -0.1, 1.5])
40
+ def test_no_note_when_the_score_is_not_a_rate(value: float | None) -> None:
41
+ assert uncertainty_note(record(value, 30)) is None
42
+
43
+
44
+ def test_no_note_without_samples() -> None:
45
+ assert uncertainty_note(record(0.5, 0)) is None
File without changes
@@ -80,3 +80,8 @@ def test_every_configuration_key_is_documented() -> None:
80
80
  if key in {"model", "temperature", "max_tokens", "timeout_s", "params"}:
81
81
  continue
82
82
  assert f"`{key}`" in text, f"{section}.{key}"
83
+
84
+
85
+ def test_the_changelog_has_an_entry_for_the_current_version() -> None:
86
+ text = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8")
87
+ assert f"## [{strop.__version__}]" in text
@@ -0,0 +1,15 @@
1
+ from strop.formatting import fmt, shorten
2
+
3
+
4
+ def test_fmt_uses_fixed_decimals_and_marks_missing_values() -> None:
5
+ assert fmt(0.5) == "0.5000"
6
+ assert fmt(0.123456, 2) == "0.12"
7
+ assert fmt(None) == "n/a"
8
+
9
+
10
+ def test_shorten_collapses_whitespace_and_marks_the_cut() -> None:
11
+ assert shorten("a\n b\tc") == "a b c"
12
+ cut = shorten("word " * 30, 20)
13
+ assert len(cut) == 20
14
+ assert cut.endswith("...")
15
+ assert shorten("short", 20) == "short"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes