strop-cli 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (210) hide show
  1. strop_cli-0.1.0/.gitignore +39 -0
  2. strop_cli-0.1.0/LICENSE +21 -0
  3. strop_cli-0.1.0/PKG-INFO +242 -0
  4. strop_cli-0.1.0/README.md +213 -0
  5. strop_cli-0.1.0/pyproject.toml +91 -0
  6. strop_cli-0.1.0/src/strop/__init__.py +5 -0
  7. strop_cli-0.1.0/src/strop/__main__.py +5 -0
  8. strop_cli-0.1.0/src/strop/analysis/__init__.py +17 -0
  9. strop_cli-0.1.0/src/strop/analysis/classify.py +52 -0
  10. strop_cli-0.1.0/src/strop/analysis/evidence.py +52 -0
  11. strop_cli-0.1.0/src/strop/analysis/facts.py +74 -0
  12. strop_cli-0.1.0/src/strop/analysis/patterns.py +90 -0
  13. strop_cli-0.1.0/src/strop/cli/__init__.py +5 -0
  14. strop_cli-0.1.0/src/strop/cli/analyze.py +93 -0
  15. strop_cli-0.1.0/src/strop/cli/app.py +82 -0
  16. strop_cli-0.1.0/src/strop/cli/data.py +111 -0
  17. strop_cli-0.1.0/src/strop/cli/evaluate.py +145 -0
  18. strop_cli-0.1.0/src/strop/cli/export.py +74 -0
  19. strop_cli-0.1.0/src/strop/cli/final.py +106 -0
  20. strop_cli-0.1.0/src/strop/cli/init.py +131 -0
  21. strop_cli-0.1.0/src/strop/cli/optimize.py +156 -0
  22. strop_cli-0.1.0/src/strop/cli/options.py +20 -0
  23. strop_cli-0.1.0/src/strop/cli/progress.py +18 -0
  24. strop_cli-0.1.0/src/strop/cli/prompts.py +110 -0
  25. strop_cli-0.1.0/src/strop/cli/status.py +68 -0
  26. strop_cli-0.1.0/src/strop/cli/versions.py +123 -0
  27. strop_cli-0.1.0/src/strop/config/__init__.py +16 -0
  28. strop_cli-0.1.0/src/strop/config/base.py +9 -0
  29. strop_cli-0.1.0/src/strop/config/constraints.py +33 -0
  30. strop_cli-0.1.0/src/strop/config/dataset.py +63 -0
  31. strop_cli-0.1.0/src/strop/config/enums.py +52 -0
  32. strop_cli-0.1.0/src/strop/config/io.py +52 -0
  33. strop_cli-0.1.0/src/strop/config/limits.py +19 -0
  34. strop_cli-0.1.0/src/strop/config/llm.py +42 -0
  35. strop_cli-0.1.0/src/strop/config/metrics.py +86 -0
  36. strop_cli-0.1.0/src/strop/config/optimization.py +23 -0
  37. strop_cli-0.1.0/src/strop/config/output.py +31 -0
  38. strop_cli-0.1.0/src/strop/config/overrides.py +22 -0
  39. strop_cli-0.1.0/src/strop/config/root.py +36 -0
  40. strop_cli-0.1.0/src/strop/config/rules.py +67 -0
  41. strop_cli-0.1.0/src/strop/config/split.py +23 -0
  42. strop_cli-0.1.0/src/strop/data/__init__.py +58 -0
  43. strop_cli-0.1.0/src/strop/data/loaders.py +85 -0
  44. strop_cli-0.1.0/src/strop/data/models.py +44 -0
  45. strop_cli-0.1.0/src/strop/data/rows.py +99 -0
  46. strop_cli-0.1.0/src/strop/data/splitting.py +103 -0
  47. strop_cli-0.1.0/src/strop/env.py +44 -0
  48. strop_cli-0.1.0/src/strop/errors.py +59 -0
  49. strop_cli-0.1.0/src/strop/evaluation/__init__.py +1 -0
  50. strop_cli-0.1.0/src/strop/evaluation/budget.py +67 -0
  51. strop_cli-0.1.0/src/strop/evaluation/budgeted.py +23 -0
  52. strop_cli-0.1.0/src/strop/evaluation/examples.py +49 -0
  53. strop_cli-0.1.0/src/strop/evaluation/models.py +105 -0
  54. strop_cli-0.1.0/src/strop/evaluation/parsing.py +96 -0
  55. strop_cli-0.1.0/src/strop/evaluation/runner.py +107 -0
  56. strop_cli-0.1.0/src/strop/formatting.py +14 -0
  57. strop_cli-0.1.0/src/strop/llm/__init__.py +7 -0
  58. strop_cli-0.1.0/src/strop/llm/caching.py +28 -0
  59. strop_cli-0.1.0/src/strop/llm/client.py +15 -0
  60. strop_cli-0.1.0/src/strop/llm/factory.py +14 -0
  61. strop_cli-0.1.0/src/strop/llm/litellm_client.py +90 -0
  62. strop_cli-0.1.0/src/strop/llm/mock.py +43 -0
  63. strop_cli-0.1.0/src/strop/llm/retry.py +44 -0
  64. strop_cli-0.1.0/src/strop/llm/structured.py +52 -0
  65. strop_cli-0.1.0/src/strop/llm/types.py +59 -0
  66. strop_cli-0.1.0/src/strop/log.py +49 -0
  67. strop_cli-0.1.0/src/strop/metrics/__init__.py +1 -0
  68. strop_cli-0.1.0/src/strop/metrics/basic.py +30 -0
  69. strop_cli-0.1.0/src/strop/metrics/classification.py +164 -0
  70. strop_cli-0.1.0/src/strop/metrics/judge.py +130 -0
  71. strop_cli-0.1.0/src/strop/metrics/python_metric.py +101 -0
  72. strop_cli-0.1.0/src/strop/metrics/python_runner.py +65 -0
  73. strop_cli-0.1.0/src/strop/metrics/registry.py +115 -0
  74. strop_cli-0.1.0/src/strop/metrics/rules.py +55 -0
  75. strop_cli-0.1.0/src/strop/metrics/structured.py +109 -0
  76. strop_cli-0.1.0/src/strop/metrics/suite.py +109 -0
  77. strop_cli-0.1.0/src/strop/metrics/text.py +55 -0
  78. strop_cli-0.1.0/src/strop/metrics/types.py +106 -0
  79. strop_cli-0.1.0/src/strop/optimization/__init__.py +25 -0
  80. strop_cli-0.1.0/src/strop/optimization/acceptance.py +170 -0
  81. strop_cli-0.1.0/src/strop/optimization/generator.py +92 -0
  82. strop_cli-0.1.0/src/strop/optimization/meta_prompt.py +101 -0
  83. strop_cli-0.1.0/src/strop/optimization/types.py +69 -0
  84. strop_cli-0.1.0/src/strop/prompts/__init__.py +18 -0
  85. strop_cli-0.1.0/src/strop/prompts/diff.py +139 -0
  86. strop_cli-0.1.0/src/strop/prompts/files.py +92 -0
  87. strop_cli-0.1.0/src/strop/prompts/models.py +51 -0
  88. strop_cli-0.1.0/src/strop/prompts/render.py +16 -0
  89. strop_cli-0.1.0/src/strop/prompts/template.py +45 -0
  90. strop_cli-0.1.0/src/strop/prompts/validation.py +31 -0
  91. strop_cli-0.1.0/src/strop/prompts/versions.py +34 -0
  92. strop_cli-0.1.0/src/strop/py.typed +0 -0
  93. strop_cli-0.1.0/src/strop/reporting/__init__.py +6 -0
  94. strop_cli-0.1.0/src/strop/reporting/console.py +20 -0
  95. strop_cli-0.1.0/src/strop/reporting/diff_view.py +102 -0
  96. strop_cli-0.1.0/src/strop/reporting/evaluation_view.py +141 -0
  97. strop_cli-0.1.0/src/strop/reporting/final_view.py +109 -0
  98. strop_cli-0.1.0/src/strop/reporting/prompt_export.py +58 -0
  99. strop_cli-0.1.0/src/strop/reporting/report_markdown.py +157 -0
  100. strop_cli-0.1.0/src/strop/reporting/run_view.py +118 -0
  101. strop_cli-0.1.0/src/strop/services/__init__.py +1 -0
  102. strop_cli-0.1.0/src/strop/services/analysis.py +124 -0
  103. strop_cli-0.1.0/src/strop/services/clean.py +38 -0
  104. strop_cli-0.1.0/src/strop/services/compare.py +60 -0
  105. strop_cli-0.1.0/src/strop/services/context.py +70 -0
  106. strop_cli-0.1.0/src/strop/services/data.py +65 -0
  107. strop_cli-0.1.0/src/strop/services/evaluate.py +105 -0
  108. strop_cli-0.1.0/src/strop/services/evaluator.py +169 -0
  109. strop_cli-0.1.0/src/strop/services/export.py +90 -0
  110. strop_cli-0.1.0/src/strop/services/final_test.py +69 -0
  111. strop_cli-0.1.0/src/strop/services/metrics.py +89 -0
  112. strop_cli-0.1.0/src/strop/services/optimize.py +187 -0
  113. strop_cli-0.1.0/src/strop/services/partitions.py +46 -0
  114. strop_cli-0.1.0/src/strop/services/project.py +55 -0
  115. strop_cli-0.1.0/src/strop/services/prompts.py +76 -0
  116. strop_cli-0.1.0/src/strop/services/report.py +194 -0
  117. strop_cli-0.1.0/src/strop/services/review.py +43 -0
  118. strop_cli-0.1.0/src/strop/services/scaffold.py +79 -0
  119. strop_cli-0.1.0/src/strop/services/turn.py +234 -0
  120. strop_cli-0.1.0/src/strop/storage/__init__.py +6 -0
  121. strop_cli-0.1.0/src/strop/storage/cache.py +45 -0
  122. strop_cli-0.1.0/src/strop/storage/database.py +45 -0
  123. strop_cli-0.1.0/src/strop/storage/datasets.py +129 -0
  124. strop_cli-0.1.0/src/strop/storage/evaluations.py +160 -0
  125. strop_cli-0.1.0/src/strop/storage/migrations/001_initial.sql +5 -0
  126. strop_cli-0.1.0/src/strop/storage/migrations/002_datasets.sql +27 -0
  127. strop_cli-0.1.0/src/strop/storage/migrations/003_prompt_versions.sql +22 -0
  128. strop_cli-0.1.0/src/strop/storage/migrations/004_llm_cache.sql +6 -0
  129. strop_cli-0.1.0/src/strop/storage/migrations/005_evaluations.sql +37 -0
  130. strop_cli-0.1.0/src/strop/storage/migrations/006_runs.sql +52 -0
  131. strop_cli-0.1.0/src/strop/storage/migrations/007_example_feedback.sql +2 -0
  132. strop_cli-0.1.0/src/strop/storage/migrations/__init__.py +5 -0
  133. strop_cli-0.1.0/src/strop/storage/migrator.py +91 -0
  134. strop_cli-0.1.0/src/strop/storage/paths.py +30 -0
  135. strop_cli-0.1.0/src/strop/storage/prompts.py +96 -0
  136. strop_cli-0.1.0/src/strop/storage/run_models.py +75 -0
  137. strop_cli-0.1.0/src/strop/storage/runs.py +263 -0
  138. strop_cli-0.1.0/tests/__init__.py +0 -0
  139. strop_cli-0.1.0/tests/conftest.py +81 -0
  140. strop_cli-0.1.0/tests/helpers.py +16 -0
  141. strop_cli-0.1.0/tests/integration/__init__.py +0 -0
  142. strop_cli-0.1.0/tests/integration/conftest.py +90 -0
  143. strop_cli-0.1.0/tests/integration/test_examples.py +60 -0
  144. strop_cli-0.1.0/tests/integration/test_export_and_clean.py +169 -0
  145. strop_cli-0.1.0/tests/integration/test_final_and_compare.py +103 -0
  146. strop_cli-0.1.0/tests/integration/test_json_task.py +129 -0
  147. strop_cli-0.1.0/tests/integration/test_litellm_offline.py +43 -0
  148. strop_cli-0.1.0/tests/integration/test_model_overrides.py +48 -0
  149. strop_cli-0.1.0/tests/integration/test_optimize_flow.py +236 -0
  150. strop_cli-0.1.0/tests/unit/__init__.py +0 -0
  151. strop_cli-0.1.0/tests/unit/analysis/__init__.py +0 -0
  152. strop_cli-0.1.0/tests/unit/analysis/test_facts.py +93 -0
  153. strop_cli-0.1.0/tests/unit/analysis/test_patterns.py +117 -0
  154. strop_cli-0.1.0/tests/unit/cli/__init__.py +0 -0
  155. strop_cli-0.1.0/tests/unit/cli/test_data.py +86 -0
  156. strop_cli-0.1.0/tests/unit/cli/test_evaluate.py +194 -0
  157. strop_cli-0.1.0/tests/unit/cli/test_init.py +96 -0
  158. strop_cli-0.1.0/tests/unit/cli/test_init_metrics.py +75 -0
  159. strop_cli-0.1.0/tests/unit/cli/test_prompts.py +164 -0
  160. strop_cli-0.1.0/tests/unit/config/__init__.py +0 -0
  161. strop_cli-0.1.0/tests/unit/config/test_constraints.py +29 -0
  162. strop_cli-0.1.0/tests/unit/config/test_io.py +62 -0
  163. strop_cli-0.1.0/tests/unit/config/test_rules.py +73 -0
  164. strop_cli-0.1.0/tests/unit/config/test_sections.py +143 -0
  165. strop_cli-0.1.0/tests/unit/data/__init__.py +0 -0
  166. strop_cli-0.1.0/tests/unit/data/test_load_dataset.py +45 -0
  167. strop_cli-0.1.0/tests/unit/data/test_loaders.py +79 -0
  168. strop_cli-0.1.0/tests/unit/data/test_rows.py +96 -0
  169. strop_cli-0.1.0/tests/unit/data/test_splitting.py +101 -0
  170. strop_cli-0.1.0/tests/unit/evaluation/__init__.py +0 -0
  171. strop_cli-0.1.0/tests/unit/evaluation/test_budget.py +65 -0
  172. strop_cli-0.1.0/tests/unit/evaluation/test_parsing.py +100 -0
  173. strop_cli-0.1.0/tests/unit/evaluation/test_runner.py +111 -0
  174. strop_cli-0.1.0/tests/unit/llm/__init__.py +0 -0
  175. strop_cli-0.1.0/tests/unit/llm/test_litellm_client.py +138 -0
  176. strop_cli-0.1.0/tests/unit/llm/test_mock.py +53 -0
  177. strop_cli-0.1.0/tests/unit/llm/test_retry_and_cache.py +131 -0
  178. strop_cli-0.1.0/tests/unit/llm/test_types.py +39 -0
  179. strop_cli-0.1.0/tests/unit/metrics/__init__.py +0 -0
  180. strop_cli-0.1.0/tests/unit/metrics/test_classification.py +94 -0
  181. strop_cli-0.1.0/tests/unit/metrics/test_judge.py +115 -0
  182. strop_cli-0.1.0/tests/unit/metrics/test_python_metric.py +130 -0
  183. strop_cli-0.1.0/tests/unit/metrics/test_structured.py +71 -0
  184. strop_cli-0.1.0/tests/unit/metrics/test_suite.py +87 -0
  185. strop_cli-0.1.0/tests/unit/metrics/test_text_and_rules.py +87 -0
  186. strop_cli-0.1.0/tests/unit/optimization/__init__.py +0 -0
  187. strop_cli-0.1.0/tests/unit/optimization/test_acceptance.py +150 -0
  188. strop_cli-0.1.0/tests/unit/optimization/test_generator.py +155 -0
  189. strop_cli-0.1.0/tests/unit/optimization/test_isolation.py +33 -0
  190. strop_cli-0.1.0/tests/unit/prompts/__init__.py +0 -0
  191. strop_cli-0.1.0/tests/unit/prompts/test_diff.py +64 -0
  192. strop_cli-0.1.0/tests/unit/prompts/test_export_formats.py +68 -0
  193. strop_cli-0.1.0/tests/unit/prompts/test_files.py +84 -0
  194. strop_cli-0.1.0/tests/unit/prompts/test_template.py +83 -0
  195. strop_cli-0.1.0/tests/unit/prompts/test_validation.py +58 -0
  196. strop_cli-0.1.0/tests/unit/services/__init__.py +0 -0
  197. strop_cli-0.1.0/tests/unit/services/test_evaluator.py +70 -0
  198. strop_cli-0.1.0/tests/unit/services/test_metric_setup.py +72 -0
  199. strop_cli-0.1.0/tests/unit/storage/__init__.py +0 -0
  200. strop_cli-0.1.0/tests/unit/storage/test_database.py +61 -0
  201. strop_cli-0.1.0/tests/unit/storage/test_datasets.py +63 -0
  202. strop_cli-0.1.0/tests/unit/storage/test_evaluations.py +113 -0
  203. strop_cli-0.1.0/tests/unit/storage/test_migrator.py +53 -0
  204. strop_cli-0.1.0/tests/unit/storage/test_prompts.py +74 -0
  205. strop_cli-0.1.0/tests/unit/storage/test_runs.py +115 -0
  206. strop_cli-0.1.0/tests/unit/test_docs.py +82 -0
  207. strop_cli-0.1.0/tests/unit/test_env.py +64 -0
  208. strop_cli-0.1.0/tests/unit/test_errors.py +34 -0
  209. strop_cli-0.1.0/tests/unit/test_log.py +48 -0
  210. strop_cli-0.1.0/tests/unit/test_package.py +7 -0
@@ -0,0 +1,39 @@
1
+ # Local-only files
2
+ .claude
3
+ CLAUDE.md
4
+
5
+ # Python
6
+ __pycache__/
7
+ *.py[cod]
8
+ *.egg-info/
9
+ .eggs/
10
+ build/
11
+ dist/
12
+
13
+ # Virtual environments
14
+ .venv/
15
+ venv/
16
+ env/
17
+
18
+ # Tool caches and reports
19
+ .pytest_cache/
20
+ .mypy_cache/
21
+ .ruff_cache/
22
+ .coverage
23
+ .coverage.*
24
+ htmlcov/
25
+ coverage.xml
26
+
27
+ # Secrets
28
+ .env
29
+ .env.*
30
+ !.env.example
31
+
32
+ # Strop runtime data (created inside user projects)
33
+ .strop/
34
+
35
+ # Editors and OS
36
+ .vscode/
37
+ .idea/
38
+ .DS_Store
39
+ Thumbs.db
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Baran Yalcin
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,242 @@
1
+ Metadata-Version: 2.5
2
+ Name: strop-cli
3
+ Version: 0.1.0
4
+ Summary: Command-line tool that improves LLM prompts by testing them against your own data.
5
+ Project-URL: Homepage, https://github.com/baranylcn/strop
6
+ Project-URL: Documentation, https://github.com/baranylcn/strop/tree/main/docs
7
+ Project-URL: Changelog, https://github.com/baranylcn/strop/blob/main/CHANGELOG.md
8
+ Project-URL: Issues, https://github.com/baranylcn/strop/issues
9
+ Author: Baran Yalcin
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ Keywords: cli,evaluation,llm,prompt,prompt-optimization
13
+ Classifier: Development Status :: 3 - Alpha
14
+ Classifier: Environment :: Console
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Typing :: Typed
22
+ Requires-Python: >=3.11
23
+ Requires-Dist: jsonschema>=4.23
24
+ Requires-Dist: litellm>=1.60
25
+ Requires-Dist: pydantic>=2.9
26
+ Requires-Dist: ruamel-yaml>=0.18
27
+ Requires-Dist: typer>=0.15
28
+ Description-Content-Type: text/markdown
29
+
30
+ # Strop
31
+
32
+ Strop is a command-line tool that improves LLM prompts by testing them against your own data. It runs your prompt, scores the outputs with a metric you define, asks a model to revise the prompt based on the failures, and keeps a revision only when the measured results get better.
33
+
34
+ ## Status
35
+
36
+ Alpha. Version 0.1.0. Commands and the `strop.yaml` format may change between releases.
37
+
38
+ ## What it does
39
+
40
+ You give Strop a prompt, a dataset, and a definition of success. Strop then repeats this loop, for as many turns as you allow:
41
+
42
+ 1. Score the current prompt on a validation set.
43
+ 2. Run it on a separate training set and collect the failures, grouped by error type.
44
+ 3. Ask an optimizer model for revised prompts that address those failures. Each revision must cite the examples that motivated it.
45
+ 4. Score every revision on the same validation rows. Keep one only if the primary metric improves and none of your constraints is broken.
46
+ 5. Start the next turn from the best prompt.
47
+
48
+ When the loop ends, you run the final prompt once on a held-out test set. Strop prints a colored diff between your original prompt and the final one.
49
+
50
+ Everything is local. Prompt versions, results, and cached model responses are stored in a `.strop/` directory inside your project. Strop has no server and no account, and it sends nothing anywhere except the model providers you configure.
51
+
52
+ ## Install
53
+
54
+ Strop needs Python 3.11 or newer.
55
+
56
+ ```bash
57
+ pip install strop-cli
58
+ strop --version
59
+ ```
60
+
61
+ The package is called `strop-cli` on PyPI because the name `strop` is reserved there. The command it installs is `strop`. To keep Strop out of your other Python environments, install it with `pipx install strop-cli` instead. To work on Strop itself, see [Development](#development).
62
+
63
+ Strop calls models through [LiteLLM](https://docs.litellm.ai/docs/providers), so you can use any provider and model it supports. Set the API key your provider expects as an environment variable, or put it in a `.env` file in your project directory. Strop has no default model; you name every model yourself.
64
+
65
+ ## Quick start
66
+
67
+ The [repository](https://github.com/baranylcn/strop) includes a ready-to-run example in `examples/ticket_routing`: 150 synthetic support tickets in five categories, with a deliberately vague starting prompt. Clone the repository to get it.
68
+
69
+ ```bash
70
+ cd examples/ticket_routing
71
+
72
+ strop init \
73
+ --name ticket-routing \
74
+ --task "Route a customer message to a support category" \
75
+ --prompt prompt.yaml \
76
+ --data data.csv --id-column ticket_id \
77
+ --input customer_message --expected category --metadata channel \
78
+ --output-type label --labels billing,refund,exchange,technical,other \
79
+ --metric macro_f1 --constraint "valid_output_rate>=0.98" \
80
+ --target-model <provider/model> --optimizer-model <provider/model>
81
+ ```
82
+
83
+ `strop init` writes `strop.yaml` and a `.strop/` directory, and tells you which providers will receive your dataset rows. Nothing is sent yet. Next, check and split the data:
84
+
85
+ ```console
86
+ $ strop data check
87
+ Dataset: data.csv
88
+ Records: 150, valid: 150, invalid: 0
89
+ Columns: ticket_id, customer_message, category, channel
90
+
91
+ $ strop data split
92
+ Created split 1 (seed 42) over 150 rows:
93
+ train 90
94
+ validation 30
95
+ test 30
96
+ ```
97
+
98
+ The split is stored, so it never changes unless you ask for a new one. Now measure the starting prompt and start the loop:
99
+
100
+ ```bash
101
+ strop baseline
102
+ strop optimize --max-turns 5
103
+ strop test --confirm
104
+ strop export
105
+ ```
106
+
107
+ These commands call your models, so their output depends on which models you choose and is not shown here. `strop optimize` prints each turn as it goes and, at the end, the metric before and after, a table of every candidate and why it was kept or rejected, and the prompt diff. `strop export` writes the final prompt and a report to `exports/`.
108
+
109
+ ## How success is measured
110
+
111
+ Strop cannot know what a correct output is. You define it, and Strop measures your definition.
112
+
113
+ | Way to define success | Use it when |
114
+ | --- | --- |
115
+ | Labeled data (`--expected`) | You have the right answer for each row: classification, extraction. |
116
+ | Rules (`--rule`) | You can check the output directly: length, required words, format. |
117
+ | A Python function (`--metric-file`) | You can compute a score in code. |
118
+ | A judge rubric (`--rubric-file`) | Quality needs judgment and there are no reference answers. |
119
+
120
+ Judge scores come from another language model, so they are estimates. Strop labels them as subjective and never mixes them with deterministic metrics. See [docs/metrics.md](https://github.com/baranylcn/strop/blob/main/docs/metrics.md) for every metric and `examples/support_replies` for rules, a custom metric, and judge rubrics together.
121
+
122
+ ## How a revision is accepted
123
+
124
+ A candidate becomes the best prompt only if the primary metric improves by a minimum amount, every constraint you set still holds, the share of valid outputs does not drop, no class loses more than a set amount of F1, and the candidate wins on more validation examples than it loses. Every failed rule is recorded with the candidate.
125
+
126
+ The test partition is locked in code. Nothing the optimizer sees comes from it, and `strop test` refuses to run without `--confirm`, or a second time without `--allow-rerun`, because tuning a prompt after looking at the test result makes the result optimistic. [docs/how-it-works.md](https://github.com/baranylcn/strop/blob/main/docs/how-it-works.md) explains the rules in detail.
127
+
128
+ ## Commands
129
+
130
+ | Command | Purpose |
131
+ | --- | --- |
132
+ | `strop init` | Create `strop.yaml`, a starter prompt, and the `.strop/` directory. |
133
+ | `strop data check` | Validate the dataset and list invalid rows with their row numbers. |
134
+ | `strop data split` | Create the train, validation, and test split. |
135
+ | `strop baseline` | Measure the initial prompt on the validation set. |
136
+ | `strop eval` | Score any prompt version on the train or validation set. Changes nothing. |
137
+ | `strop analyze` | Count and list how a prompt fails on the training data. |
138
+ | `strop optimize` | Run the optimization loop. Resumable. |
139
+ | `strop approve`, `strop reject` | Decide a candidate when running with `--approval manual`. |
140
+ | `strop test` | Run the final evaluation on the held-out test set. |
141
+ | `strop compare` | Run one prompt on several models and compare the results. |
142
+ | `strop diff`, `strop history` | Compare prompt versions and see every decision. |
143
+ | `strop prompt add`, `strop prompt show` | Save your own edit as a version, or print a version. |
144
+ | `strop rollback` | Make an earlier version the best prompt. Nothing is deleted. |
145
+ | `strop status` | Show the split, the best prompt, and recent evaluations. |
146
+ | `strop export` | Write the prompt, a JSON and Markdown report, and per-example results. |
147
+ | `strop clean` | Delete stored cache, outputs, or the whole `.strop/` directory. |
148
+
149
+ Run `strop <command> --help` for the options of each command. The commands never ask questions: every choice is a flag or a value in `strop.yaml`, so they work in scripts. Commands that print results accept `--json` for machine-readable output, and every command accepts `--dir` to name the project directory.
150
+
151
+ ## Configuration
152
+
153
+ `strop init` writes `strop.yaml`. This is a minimal one:
154
+
155
+ ```yaml
156
+ project:
157
+ name: ticket-routing
158
+ task_description: Route a customer message to a support category
159
+ dataset:
160
+ path: data.csv
161
+ columns: {customer_message: input, category: expected}
162
+ output:
163
+ type: label
164
+ labels: [billing, refund, exchange, technical, other]
165
+ models:
166
+ target: {model: <provider/model>}
167
+ optimizer: {model: <provider/model>}
168
+ metrics:
169
+ primary: macro_f1
170
+ ```
171
+
172
+ The prompt itself is a separate file with chat messages and a list of the messages the optimizer may change:
173
+
174
+ ```yaml
175
+ messages:
176
+ - role: system
177
+ content: |-
178
+ You route customer messages to a support team.
179
+ Reply with the category only.
180
+ - role: user
181
+ content: |-
182
+ Message: {{customer_message}}
183
+ optimize: [system]
184
+ ```
185
+
186
+ The optimizer never touches messages outside `optimize`, never changes `{{variables}}`, and never changes text wrapped in `<strop:fixed>...</strop:fixed>`. Your file is not overwritten; revisions are stored as versions. The full reference is in [docs/configuration.md](https://github.com/baranylcn/strop/blob/main/docs/configuration.md).
187
+
188
+ ## Models and providers
189
+
190
+ The roles are `target` (the model your prompt is for), `optimizer` (proposes revisions), `analyzer` (proposes failure patterns, defaults to the optimizer), and `judge` (scores rubrics). Each can be a different provider and model. Model names are LiteLLM names, such as `<provider>/<model>`. Pass `--optimizer-model` or `--analyzer-model` to `strop optimize` to change a role for one run, and use `strop compare --models a,b` to compare target models on the same rows.
191
+
192
+ Strop retries temporary provider errors with exponential backoff. A call that still fails is recorded as an error for that one example and does not stop the run.
193
+
194
+ ## Data and privacy
195
+
196
+ - Dataset rows go to the providers of the models you configure. The target model receives every row it is run on. The optimizer and analyzer receive a bounded sample of failed train examples. The judge receives each example it scores. `strop init` lists the providers.
197
+ - `evidence_examples` and `evidence_chars` in `strop.yaml` limit how much of the failed examples is shown to the optimizer and analyzer.
198
+ - Dataset content is always passed to models as quoted data, and the meta-prompts tell the models not to follow instructions found inside it.
199
+ - API keys are read from the environment or `.env` and are never stored by Strop. Key-like strings are removed from logs and provider error messages.
200
+ - `.strop/` holds model outputs and cached responses. Keep it and `.env` out of version control. `strop clean` deletes what you choose.
201
+ - A `metric.py` file is your own code and runs as written, in a separate process. Strop never executes anything found in a dataset.
202
+
203
+ ## Exit codes
204
+
205
+ | Code | Meaning |
206
+ | --- | --- |
207
+ | 0 | Success. |
208
+ | 1 | The command failed. |
209
+ | 2 | Usage or configuration error. |
210
+ | 10 | `strop optimize` paused and waits for `strop approve` or `strop reject`. |
211
+ | 11 | `strop optimize` stopped because a budget was reached. |
212
+ | 12 | The command was interrupted. Progress is saved. |
213
+
214
+ ## Limitations
215
+
216
+ - A small validation set gives noisy comparisons. Strop warns below 30 validation rows and refuses to optimize below 10.
217
+ - The test set is only a fair estimate if you look at it once.
218
+ - Judge metrics are estimates, and a judge model can prefer its own style.
219
+ - The optimizer is a language model. It proposes revisions that fit the failures it is shown, and the validation rules decide whether they help. A revision that passes can still be tuned to the validation set, which is what the test set is for.
220
+ - Results depend on the models you use. A prompt tuned for one model may not transfer to another.
221
+ - Token and cost data come from the provider. When a provider reports none, Strop shows it as unavailable instead of estimating it.
222
+ - Strop optimizes single prompts, not agents or tool chains.
223
+
224
+ ## Development
225
+
226
+ Strop uses [uv](https://docs.astral.sh/uv/).
227
+
228
+ ```bash
229
+ git clone https://github.com/baranylcn/strop.git
230
+ cd strop
231
+ uv sync
232
+ uv run pytest
233
+ uv run ruff check .
234
+ uv run ruff format --check .
235
+ uv run mypy
236
+ ```
237
+
238
+ The tests use scripted models and make no network calls. See [CONTRIBUTING.md](https://github.com/baranylcn/strop/blob/main/CONTRIBUTING.md) for the project layout and conventions.
239
+
240
+ ## License
241
+
242
+ MIT. See [LICENSE](https://github.com/baranylcn/strop/blob/main/LICENSE).
@@ -0,0 +1,213 @@
1
+ # Strop
2
+
3
+ Strop is a command-line tool that improves LLM prompts by testing them against your own data. It runs your prompt, scores the outputs with a metric you define, asks a model to revise the prompt based on the failures, and keeps a revision only when the measured results get better.
4
+
5
+ ## Status
6
+
7
+ Alpha. Version 0.1.0. Commands and the `strop.yaml` format may change between releases.
8
+
9
+ ## What it does
10
+
11
+ You give Strop a prompt, a dataset, and a definition of success. Strop then repeats this loop, for as many turns as you allow:
12
+
13
+ 1. Score the current prompt on a validation set.
14
+ 2. Run it on a separate training set and collect the failures, grouped by error type.
15
+ 3. Ask an optimizer model for revised prompts that address those failures. Each revision must cite the examples that motivated it.
16
+ 4. Score every revision on the same validation rows. Keep one only if the primary metric improves and none of your constraints is broken.
17
+ 5. Start the next turn from the best prompt.
18
+
19
+ When the loop ends, you run the final prompt once on a held-out test set. Strop prints a colored diff between your original prompt and the final one.
20
+
21
+ Everything is local. Prompt versions, results, and cached model responses are stored in a `.strop/` directory inside your project. Strop has no server and no account, and it sends nothing anywhere except the model providers you configure.
22
+
23
+ ## Install
24
+
25
+ Strop needs Python 3.11 or newer.
26
+
27
+ ```bash
28
+ pip install strop-cli
29
+ strop --version
30
+ ```
31
+
32
+ The package is called `strop-cli` on PyPI because the name `strop` is reserved there. The command it installs is `strop`. To keep Strop out of your other Python environments, install it with `pipx install strop-cli` instead. To work on Strop itself, see [Development](#development).
33
+
34
+ Strop calls models through [LiteLLM](https://docs.litellm.ai/docs/providers), so you can use any provider and model it supports. Set the API key your provider expects as an environment variable, or put it in a `.env` file in your project directory. Strop has no default model; you name every model yourself.
35
+
36
+ ## Quick start
37
+
38
+ The [repository](https://github.com/baranylcn/strop) includes a ready-to-run example in `examples/ticket_routing`: 150 synthetic support tickets in five categories, with a deliberately vague starting prompt. Clone the repository to get it.
39
+
40
+ ```bash
41
+ cd examples/ticket_routing
42
+
43
+ strop init \
44
+ --name ticket-routing \
45
+ --task "Route a customer message to a support category" \
46
+ --prompt prompt.yaml \
47
+ --data data.csv --id-column ticket_id \
48
+ --input customer_message --expected category --metadata channel \
49
+ --output-type label --labels billing,refund,exchange,technical,other \
50
+ --metric macro_f1 --constraint "valid_output_rate>=0.98" \
51
+ --target-model <provider/model> --optimizer-model <provider/model>
52
+ ```
53
+
54
+ `strop init` writes `strop.yaml` and a `.strop/` directory, and tells you which providers will receive your dataset rows. Nothing is sent yet. Next, check and split the data:
55
+
56
+ ```console
57
+ $ strop data check
58
+ Dataset: data.csv
59
+ Records: 150, valid: 150, invalid: 0
60
+ Columns: ticket_id, customer_message, category, channel
61
+
62
+ $ strop data split
63
+ Created split 1 (seed 42) over 150 rows:
64
+ train 90
65
+ validation 30
66
+ test 30
67
+ ```
68
+
69
+ The split is stored, so it never changes unless you ask for a new one. Now measure the starting prompt and start the loop:
70
+
71
+ ```bash
72
+ strop baseline
73
+ strop optimize --max-turns 5
74
+ strop test --confirm
75
+ strop export
76
+ ```
77
+
78
+ These commands call your models, so their output depends on which models you choose and is not shown here. `strop optimize` prints each turn as it goes and, at the end, the metric before and after, a table of every candidate and why it was kept or rejected, and the prompt diff. `strop export` writes the final prompt and a report to `exports/`.
79
+
80
+ ## How success is measured
81
+
82
+ Strop cannot know what a correct output is. You define it, and Strop measures your definition.
83
+
84
+ | Way to define success | Use it when |
85
+ | --- | --- |
86
+ | Labeled data (`--expected`) | You have the right answer for each row: classification, extraction. |
87
+ | Rules (`--rule`) | You can check the output directly: length, required words, format. |
88
+ | A Python function (`--metric-file`) | You can compute a score in code. |
89
+ | A judge rubric (`--rubric-file`) | Quality needs judgment and there are no reference answers. |
90
+
91
+ Judge scores come from another language model, so they are estimates. Strop labels them as subjective and never mixes them with deterministic metrics. See [docs/metrics.md](https://github.com/baranylcn/strop/blob/main/docs/metrics.md) for every metric and `examples/support_replies` for rules, a custom metric, and judge rubrics together.
92
+
93
+ ## How a revision is accepted
94
+
95
+ A candidate becomes the best prompt only if the primary metric improves by a minimum amount, every constraint you set still holds, the share of valid outputs does not drop, no class loses more than a set amount of F1, and the candidate wins on more validation examples than it loses. Every failed rule is recorded with the candidate.
96
+
97
+ The test partition is locked in code. Nothing the optimizer sees comes from it, and `strop test` refuses to run without `--confirm`, or a second time without `--allow-rerun`, because tuning a prompt after looking at the test result makes the result optimistic. [docs/how-it-works.md](https://github.com/baranylcn/strop/blob/main/docs/how-it-works.md) explains the rules in detail.
98
+
99
+ ## Commands
100
+
101
+ | Command | Purpose |
102
+ | --- | --- |
103
+ | `strop init` | Create `strop.yaml`, a starter prompt, and the `.strop/` directory. |
104
+ | `strop data check` | Validate the dataset and list invalid rows with their row numbers. |
105
+ | `strop data split` | Create the train, validation, and test split. |
106
+ | `strop baseline` | Measure the initial prompt on the validation set. |
107
+ | `strop eval` | Score any prompt version on the train or validation set. Changes nothing. |
108
+ | `strop analyze` | Count and list how a prompt fails on the training data. |
109
+ | `strop optimize` | Run the optimization loop. Resumable. |
110
+ | `strop approve`, `strop reject` | Decide a candidate when running with `--approval manual`. |
111
+ | `strop test` | Run the final evaluation on the held-out test set. |
112
+ | `strop compare` | Run one prompt on several models and compare the results. |
113
+ | `strop diff`, `strop history` | Compare prompt versions and see every decision. |
114
+ | `strop prompt add`, `strop prompt show` | Save your own edit as a version, or print a version. |
115
+ | `strop rollback` | Make an earlier version the best prompt. Nothing is deleted. |
116
+ | `strop status` | Show the split, the best prompt, and recent evaluations. |
117
+ | `strop export` | Write the prompt, a JSON and Markdown report, and per-example results. |
118
+ | `strop clean` | Delete stored cache, outputs, or the whole `.strop/` directory. |
119
+
120
+ Run `strop <command> --help` for the options of each command. The commands never ask questions: every choice is a flag or a value in `strop.yaml`, so they work in scripts. Commands that print results accept `--json` for machine-readable output, and every command accepts `--dir` to name the project directory.
121
+
122
+ ## Configuration
123
+
124
+ `strop init` writes `strop.yaml`. This is a minimal one:
125
+
126
+ ```yaml
127
+ project:
128
+ name: ticket-routing
129
+ task_description: Route a customer message to a support category
130
+ dataset:
131
+ path: data.csv
132
+ columns: {customer_message: input, category: expected}
133
+ output:
134
+ type: label
135
+ labels: [billing, refund, exchange, technical, other]
136
+ models:
137
+ target: {model: <provider/model>}
138
+ optimizer: {model: <provider/model>}
139
+ metrics:
140
+ primary: macro_f1
141
+ ```
142
+
143
+ The prompt itself is a separate file with chat messages and a list of the messages the optimizer may change:
144
+
145
+ ```yaml
146
+ messages:
147
+ - role: system
148
+ content: |-
149
+ You route customer messages to a support team.
150
+ Reply with the category only.
151
+ - role: user
152
+ content: |-
153
+ Message: {{customer_message}}
154
+ optimize: [system]
155
+ ```
156
+
157
+ The optimizer never touches messages outside `optimize`, never changes `{{variables}}`, and never changes text wrapped in `<strop:fixed>...</strop:fixed>`. Your file is not overwritten; revisions are stored as versions. The full reference is in [docs/configuration.md](https://github.com/baranylcn/strop/blob/main/docs/configuration.md).
158
+
159
+ ## Models and providers
160
+
161
+ The roles are `target` (the model your prompt is for), `optimizer` (proposes revisions), `analyzer` (proposes failure patterns, defaults to the optimizer), and `judge` (scores rubrics). Each can be a different provider and model. Model names are LiteLLM names, such as `<provider>/<model>`. Pass `--optimizer-model` or `--analyzer-model` to `strop optimize` to change a role for one run, and use `strop compare --models a,b` to compare target models on the same rows.
162
+
163
+ Strop retries temporary provider errors with exponential backoff. A call that still fails is recorded as an error for that one example and does not stop the run.
164
+
165
+ ## Data and privacy
166
+
167
+ - Dataset rows go to the providers of the models you configure. The target model receives every row it is run on. The optimizer and analyzer receive a bounded sample of failed train examples. The judge receives each example it scores. `strop init` lists the providers.
168
+ - `evidence_examples` and `evidence_chars` in `strop.yaml` limit how much of the failed examples is shown to the optimizer and analyzer.
169
+ - Dataset content is always passed to models as quoted data, and the meta-prompts tell the models not to follow instructions found inside it.
170
+ - API keys are read from the environment or `.env` and are never stored by Strop. Key-like strings are removed from logs and provider error messages.
171
+ - `.strop/` holds model outputs and cached responses. Keep it and `.env` out of version control. `strop clean` deletes what you choose.
172
+ - A `metric.py` file is your own code and runs as written, in a separate process. Strop never executes anything found in a dataset.
173
+
174
+ ## Exit codes
175
+
176
+ | Code | Meaning |
177
+ | --- | --- |
178
+ | 0 | Success. |
179
+ | 1 | The command failed. |
180
+ | 2 | Usage or configuration error. |
181
+ | 10 | `strop optimize` paused and waits for `strop approve` or `strop reject`. |
182
+ | 11 | `strop optimize` stopped because a budget was reached. |
183
+ | 12 | The command was interrupted. Progress is saved. |
184
+
185
+ ## Limitations
186
+
187
+ - A small validation set gives noisy comparisons. Strop warns below 30 validation rows and refuses to optimize below 10.
188
+ - The test set is only a fair estimate if you look at it once.
189
+ - Judge metrics are estimates, and a judge model can prefer its own style.
190
+ - The optimizer is a language model. It proposes revisions that fit the failures it is shown, and the validation rules decide whether they help. A revision that passes can still be tuned to the validation set, which is what the test set is for.
191
+ - Results depend on the models you use. A prompt tuned for one model may not transfer to another.
192
+ - Token and cost data come from the provider. When a provider reports none, Strop shows it as unavailable instead of estimating it.
193
+ - Strop optimizes single prompts, not agents or tool chains.
194
+
195
+ ## Development
196
+
197
+ Strop uses [uv](https://docs.astral.sh/uv/).
198
+
199
+ ```bash
200
+ git clone https://github.com/baranylcn/strop.git
201
+ cd strop
202
+ uv sync
203
+ uv run pytest
204
+ uv run ruff check .
205
+ uv run ruff format --check .
206
+ uv run mypy
207
+ ```
208
+
209
+ The tests use scripted models and make no network calls. See [CONTRIBUTING.md](https://github.com/baranylcn/strop/blob/main/CONTRIBUTING.md) for the project layout and conventions.
210
+
211
+ ## License
212
+
213
+ MIT. See [LICENSE](https://github.com/baranylcn/strop/blob/main/LICENSE).
@@ -0,0 +1,91 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "strop-cli"
7
+ dynamic = ["version"]
8
+ description = "Command-line tool that improves LLM prompts by testing them against your own data."
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+ license = "MIT"
12
+ license-files = ["LICENSE"]
13
+ authors = [{ name = "Baran Yalcin" }]
14
+ keywords = ["llm", "prompt", "prompt-optimization", "evaluation", "cli"]
15
+ classifiers = [
16
+ "Development Status :: 3 - Alpha",
17
+ "Environment :: Console",
18
+ "Intended Audience :: Developers",
19
+ "Operating System :: OS Independent",
20
+ "Programming Language :: Python :: 3",
21
+ "Programming Language :: Python :: 3.11",
22
+ "Programming Language :: Python :: 3.12",
23
+ "Programming Language :: Python :: 3.13",
24
+ "Typing :: Typed",
25
+ ]
26
+ dependencies = [
27
+ "jsonschema>=4.23",
28
+ "litellm>=1.60",
29
+ "pydantic>=2.9",
30
+ "ruamel-yaml>=0.18",
31
+ "typer>=0.15",
32
+ ]
33
+
34
+ [project.scripts]
35
+ strop = "strop.cli:main"
36
+
37
+ [project.urls]
38
+ Homepage = "https://github.com/baranylcn/strop"
39
+ Documentation = "https://github.com/baranylcn/strop/tree/main/docs"
40
+ Changelog = "https://github.com/baranylcn/strop/blob/main/CHANGELOG.md"
41
+ Issues = "https://github.com/baranylcn/strop/issues"
42
+
43
+ [dependency-groups]
44
+ dev = [
45
+ "mypy>=1.13",
46
+ "pytest>=8.3",
47
+ "pytest-asyncio>=0.24",
48
+ "ruff>=0.8",
49
+ "types-jsonschema>=4.26.0.20261006",
50
+ ]
51
+
52
+ [tool.hatch.version]
53
+ path = "src/strop/__init__.py"
54
+
55
+ [tool.hatch.build.targets.wheel]
56
+ packages = ["src/strop"]
57
+
58
+ [tool.hatch.build.targets.sdist]
59
+ include = ["/src", "/tests", "/README.md", "/LICENSE", "/pyproject.toml"]
60
+
61
+ [tool.ruff]
62
+ line-length = 100
63
+ target-version = "py311"
64
+ src = ["src", "tests"]
65
+
66
+ [tool.ruff.format]
67
+ line-ending = "lf"
68
+
69
+ [tool.ruff.lint]
70
+ select = ["E", "F", "W", "I", "N", "UP", "B", "C4", "SIM", "RUF", "PT", "ASYNC", "PL"]
71
+ ignore = ["PLR2004"]
72
+
73
+ [tool.ruff.lint.per-file-ignores]
74
+ "src/strop/cli/*" = ["PLR0913", "PLR0917"]
75
+
76
+ [tool.ruff.lint.pylint]
77
+ max-args = 7
78
+ max-statements = 40
79
+
80
+ [tool.mypy]
81
+ python_version = "3.11"
82
+ strict = true
83
+ files = ["src", "tests"]
84
+ plugins = ["pydantic.mypy"]
85
+
86
+ [tool.pytest.ini_options]
87
+ testpaths = ["tests"]
88
+ asyncio_mode = "auto"
89
+ asyncio_default_fixture_loop_scope = "function"
90
+ pythonpath = ["."]
91
+ addopts = "-ra --strict-markers --strict-config --import-mode=importlib"
@@ -0,0 +1,5 @@
1
+ """Strop: improve LLM prompts by testing them against your own data."""
2
+
3
+ __version__ = "0.1.0"
4
+
5
+ __all__ = ["__version__"]
@@ -0,0 +1,5 @@
1
+ """Allow ``python -m strop``."""
2
+
3
+ from strop.cli import main
4
+
5
+ main()
@@ -0,0 +1,17 @@
1
+ """Failure analysis: deterministic facts first, LLM-proposed patterns as hypotheses."""
2
+
3
+ from strop.analysis.classify import classify_outcome
4
+ from strop.analysis.evidence import evidence_payload, select_evidence
5
+ from strop.analysis.facts import AnalysisFacts, FailedExample, build_facts
6
+ from strop.analysis.patterns import Pattern, find_patterns
7
+
8
+ __all__ = [
9
+ "AnalysisFacts",
10
+ "FailedExample",
11
+ "Pattern",
12
+ "build_facts",
13
+ "classify_outcome",
14
+ "evidence_payload",
15
+ "find_patterns",
16
+ "select_evidence",
17
+ ]