strop-cli 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- strop_cli-0.1.0/.gitignore +39 -0
- strop_cli-0.1.0/LICENSE +21 -0
- strop_cli-0.1.0/PKG-INFO +242 -0
- strop_cli-0.1.0/README.md +213 -0
- strop_cli-0.1.0/pyproject.toml +91 -0
- strop_cli-0.1.0/src/strop/__init__.py +5 -0
- strop_cli-0.1.0/src/strop/__main__.py +5 -0
- strop_cli-0.1.0/src/strop/analysis/__init__.py +17 -0
- strop_cli-0.1.0/src/strop/analysis/classify.py +52 -0
- strop_cli-0.1.0/src/strop/analysis/evidence.py +52 -0
- strop_cli-0.1.0/src/strop/analysis/facts.py +74 -0
- strop_cli-0.1.0/src/strop/analysis/patterns.py +90 -0
- strop_cli-0.1.0/src/strop/cli/__init__.py +5 -0
- strop_cli-0.1.0/src/strop/cli/analyze.py +93 -0
- strop_cli-0.1.0/src/strop/cli/app.py +82 -0
- strop_cli-0.1.0/src/strop/cli/data.py +111 -0
- strop_cli-0.1.0/src/strop/cli/evaluate.py +145 -0
- strop_cli-0.1.0/src/strop/cli/export.py +74 -0
- strop_cli-0.1.0/src/strop/cli/final.py +106 -0
- strop_cli-0.1.0/src/strop/cli/init.py +131 -0
- strop_cli-0.1.0/src/strop/cli/optimize.py +156 -0
- strop_cli-0.1.0/src/strop/cli/options.py +20 -0
- strop_cli-0.1.0/src/strop/cli/progress.py +18 -0
- strop_cli-0.1.0/src/strop/cli/prompts.py +110 -0
- strop_cli-0.1.0/src/strop/cli/status.py +68 -0
- strop_cli-0.1.0/src/strop/cli/versions.py +123 -0
- strop_cli-0.1.0/src/strop/config/__init__.py +16 -0
- strop_cli-0.1.0/src/strop/config/base.py +9 -0
- strop_cli-0.1.0/src/strop/config/constraints.py +33 -0
- strop_cli-0.1.0/src/strop/config/dataset.py +63 -0
- strop_cli-0.1.0/src/strop/config/enums.py +52 -0
- strop_cli-0.1.0/src/strop/config/io.py +52 -0
- strop_cli-0.1.0/src/strop/config/limits.py +19 -0
- strop_cli-0.1.0/src/strop/config/llm.py +42 -0
- strop_cli-0.1.0/src/strop/config/metrics.py +86 -0
- strop_cli-0.1.0/src/strop/config/optimization.py +23 -0
- strop_cli-0.1.0/src/strop/config/output.py +31 -0
- strop_cli-0.1.0/src/strop/config/overrides.py +22 -0
- strop_cli-0.1.0/src/strop/config/root.py +36 -0
- strop_cli-0.1.0/src/strop/config/rules.py +67 -0
- strop_cli-0.1.0/src/strop/config/split.py +23 -0
- strop_cli-0.1.0/src/strop/data/__init__.py +58 -0
- strop_cli-0.1.0/src/strop/data/loaders.py +85 -0
- strop_cli-0.1.0/src/strop/data/models.py +44 -0
- strop_cli-0.1.0/src/strop/data/rows.py +99 -0
- strop_cli-0.1.0/src/strop/data/splitting.py +103 -0
- strop_cli-0.1.0/src/strop/env.py +44 -0
- strop_cli-0.1.0/src/strop/errors.py +59 -0
- strop_cli-0.1.0/src/strop/evaluation/__init__.py +1 -0
- strop_cli-0.1.0/src/strop/evaluation/budget.py +67 -0
- strop_cli-0.1.0/src/strop/evaluation/budgeted.py +23 -0
- strop_cli-0.1.0/src/strop/evaluation/examples.py +49 -0
- strop_cli-0.1.0/src/strop/evaluation/models.py +105 -0
- strop_cli-0.1.0/src/strop/evaluation/parsing.py +96 -0
- strop_cli-0.1.0/src/strop/evaluation/runner.py +107 -0
- strop_cli-0.1.0/src/strop/formatting.py +14 -0
- strop_cli-0.1.0/src/strop/llm/__init__.py +7 -0
- strop_cli-0.1.0/src/strop/llm/caching.py +28 -0
- strop_cli-0.1.0/src/strop/llm/client.py +15 -0
- strop_cli-0.1.0/src/strop/llm/factory.py +14 -0
- strop_cli-0.1.0/src/strop/llm/litellm_client.py +90 -0
- strop_cli-0.1.0/src/strop/llm/mock.py +43 -0
- strop_cli-0.1.0/src/strop/llm/retry.py +44 -0
- strop_cli-0.1.0/src/strop/llm/structured.py +52 -0
- strop_cli-0.1.0/src/strop/llm/types.py +59 -0
- strop_cli-0.1.0/src/strop/log.py +49 -0
- strop_cli-0.1.0/src/strop/metrics/__init__.py +1 -0
- strop_cli-0.1.0/src/strop/metrics/basic.py +30 -0
- strop_cli-0.1.0/src/strop/metrics/classification.py +164 -0
- strop_cli-0.1.0/src/strop/metrics/judge.py +130 -0
- strop_cli-0.1.0/src/strop/metrics/python_metric.py +101 -0
- strop_cli-0.1.0/src/strop/metrics/python_runner.py +65 -0
- strop_cli-0.1.0/src/strop/metrics/registry.py +115 -0
- strop_cli-0.1.0/src/strop/metrics/rules.py +55 -0
- strop_cli-0.1.0/src/strop/metrics/structured.py +109 -0
- strop_cli-0.1.0/src/strop/metrics/suite.py +109 -0
- strop_cli-0.1.0/src/strop/metrics/text.py +55 -0
- strop_cli-0.1.0/src/strop/metrics/types.py +106 -0
- strop_cli-0.1.0/src/strop/optimization/__init__.py +25 -0
- strop_cli-0.1.0/src/strop/optimization/acceptance.py +170 -0
- strop_cli-0.1.0/src/strop/optimization/generator.py +92 -0
- strop_cli-0.1.0/src/strop/optimization/meta_prompt.py +101 -0
- strop_cli-0.1.0/src/strop/optimization/types.py +69 -0
- strop_cli-0.1.0/src/strop/prompts/__init__.py +18 -0
- strop_cli-0.1.0/src/strop/prompts/diff.py +139 -0
- strop_cli-0.1.0/src/strop/prompts/files.py +92 -0
- strop_cli-0.1.0/src/strop/prompts/models.py +51 -0
- strop_cli-0.1.0/src/strop/prompts/render.py +16 -0
- strop_cli-0.1.0/src/strop/prompts/template.py +45 -0
- strop_cli-0.1.0/src/strop/prompts/validation.py +31 -0
- strop_cli-0.1.0/src/strop/prompts/versions.py +34 -0
- strop_cli-0.1.0/src/strop/py.typed +0 -0
- strop_cli-0.1.0/src/strop/reporting/__init__.py +6 -0
- strop_cli-0.1.0/src/strop/reporting/console.py +20 -0
- strop_cli-0.1.0/src/strop/reporting/diff_view.py +102 -0
- strop_cli-0.1.0/src/strop/reporting/evaluation_view.py +141 -0
- strop_cli-0.1.0/src/strop/reporting/final_view.py +109 -0
- strop_cli-0.1.0/src/strop/reporting/prompt_export.py +58 -0
- strop_cli-0.1.0/src/strop/reporting/report_markdown.py +157 -0
- strop_cli-0.1.0/src/strop/reporting/run_view.py +118 -0
- strop_cli-0.1.0/src/strop/services/__init__.py +1 -0
- strop_cli-0.1.0/src/strop/services/analysis.py +124 -0
- strop_cli-0.1.0/src/strop/services/clean.py +38 -0
- strop_cli-0.1.0/src/strop/services/compare.py +60 -0
- strop_cli-0.1.0/src/strop/services/context.py +70 -0
- strop_cli-0.1.0/src/strop/services/data.py +65 -0
- strop_cli-0.1.0/src/strop/services/evaluate.py +105 -0
- strop_cli-0.1.0/src/strop/services/evaluator.py +169 -0
- strop_cli-0.1.0/src/strop/services/export.py +90 -0
- strop_cli-0.1.0/src/strop/services/final_test.py +69 -0
- strop_cli-0.1.0/src/strop/services/metrics.py +89 -0
- strop_cli-0.1.0/src/strop/services/optimize.py +187 -0
- strop_cli-0.1.0/src/strop/services/partitions.py +46 -0
- strop_cli-0.1.0/src/strop/services/project.py +55 -0
- strop_cli-0.1.0/src/strop/services/prompts.py +76 -0
- strop_cli-0.1.0/src/strop/services/report.py +194 -0
- strop_cli-0.1.0/src/strop/services/review.py +43 -0
- strop_cli-0.1.0/src/strop/services/scaffold.py +79 -0
- strop_cli-0.1.0/src/strop/services/turn.py +234 -0
- strop_cli-0.1.0/src/strop/storage/__init__.py +6 -0
- strop_cli-0.1.0/src/strop/storage/cache.py +45 -0
- strop_cli-0.1.0/src/strop/storage/database.py +45 -0
- strop_cli-0.1.0/src/strop/storage/datasets.py +129 -0
- strop_cli-0.1.0/src/strop/storage/evaluations.py +160 -0
- strop_cli-0.1.0/src/strop/storage/migrations/001_initial.sql +5 -0
- strop_cli-0.1.0/src/strop/storage/migrations/002_datasets.sql +27 -0
- strop_cli-0.1.0/src/strop/storage/migrations/003_prompt_versions.sql +22 -0
- strop_cli-0.1.0/src/strop/storage/migrations/004_llm_cache.sql +6 -0
- strop_cli-0.1.0/src/strop/storage/migrations/005_evaluations.sql +37 -0
- strop_cli-0.1.0/src/strop/storage/migrations/006_runs.sql +52 -0
- strop_cli-0.1.0/src/strop/storage/migrations/007_example_feedback.sql +2 -0
- strop_cli-0.1.0/src/strop/storage/migrations/__init__.py +5 -0
- strop_cli-0.1.0/src/strop/storage/migrator.py +91 -0
- strop_cli-0.1.0/src/strop/storage/paths.py +30 -0
- strop_cli-0.1.0/src/strop/storage/prompts.py +96 -0
- strop_cli-0.1.0/src/strop/storage/run_models.py +75 -0
- strop_cli-0.1.0/src/strop/storage/runs.py +263 -0
- strop_cli-0.1.0/tests/__init__.py +0 -0
- strop_cli-0.1.0/tests/conftest.py +81 -0
- strop_cli-0.1.0/tests/helpers.py +16 -0
- strop_cli-0.1.0/tests/integration/__init__.py +0 -0
- strop_cli-0.1.0/tests/integration/conftest.py +90 -0
- strop_cli-0.1.0/tests/integration/test_examples.py +60 -0
- strop_cli-0.1.0/tests/integration/test_export_and_clean.py +169 -0
- strop_cli-0.1.0/tests/integration/test_final_and_compare.py +103 -0
- strop_cli-0.1.0/tests/integration/test_json_task.py +129 -0
- strop_cli-0.1.0/tests/integration/test_litellm_offline.py +43 -0
- strop_cli-0.1.0/tests/integration/test_model_overrides.py +48 -0
- strop_cli-0.1.0/tests/integration/test_optimize_flow.py +236 -0
- strop_cli-0.1.0/tests/unit/__init__.py +0 -0
- strop_cli-0.1.0/tests/unit/analysis/__init__.py +0 -0
- strop_cli-0.1.0/tests/unit/analysis/test_facts.py +93 -0
- strop_cli-0.1.0/tests/unit/analysis/test_patterns.py +117 -0
- strop_cli-0.1.0/tests/unit/cli/__init__.py +0 -0
- strop_cli-0.1.0/tests/unit/cli/test_data.py +86 -0
- strop_cli-0.1.0/tests/unit/cli/test_evaluate.py +194 -0
- strop_cli-0.1.0/tests/unit/cli/test_init.py +96 -0
- strop_cli-0.1.0/tests/unit/cli/test_init_metrics.py +75 -0
- strop_cli-0.1.0/tests/unit/cli/test_prompts.py +164 -0
- strop_cli-0.1.0/tests/unit/config/__init__.py +0 -0
- strop_cli-0.1.0/tests/unit/config/test_constraints.py +29 -0
- strop_cli-0.1.0/tests/unit/config/test_io.py +62 -0
- strop_cli-0.1.0/tests/unit/config/test_rules.py +73 -0
- strop_cli-0.1.0/tests/unit/config/test_sections.py +143 -0
- strop_cli-0.1.0/tests/unit/data/__init__.py +0 -0
- strop_cli-0.1.0/tests/unit/data/test_load_dataset.py +45 -0
- strop_cli-0.1.0/tests/unit/data/test_loaders.py +79 -0
- strop_cli-0.1.0/tests/unit/data/test_rows.py +96 -0
- strop_cli-0.1.0/tests/unit/data/test_splitting.py +101 -0
- strop_cli-0.1.0/tests/unit/evaluation/__init__.py +0 -0
- strop_cli-0.1.0/tests/unit/evaluation/test_budget.py +65 -0
- strop_cli-0.1.0/tests/unit/evaluation/test_parsing.py +100 -0
- strop_cli-0.1.0/tests/unit/evaluation/test_runner.py +111 -0
- strop_cli-0.1.0/tests/unit/llm/__init__.py +0 -0
- strop_cli-0.1.0/tests/unit/llm/test_litellm_client.py +138 -0
- strop_cli-0.1.0/tests/unit/llm/test_mock.py +53 -0
- strop_cli-0.1.0/tests/unit/llm/test_retry_and_cache.py +131 -0
- strop_cli-0.1.0/tests/unit/llm/test_types.py +39 -0
- strop_cli-0.1.0/tests/unit/metrics/__init__.py +0 -0
- strop_cli-0.1.0/tests/unit/metrics/test_classification.py +94 -0
- strop_cli-0.1.0/tests/unit/metrics/test_judge.py +115 -0
- strop_cli-0.1.0/tests/unit/metrics/test_python_metric.py +130 -0
- strop_cli-0.1.0/tests/unit/metrics/test_structured.py +71 -0
- strop_cli-0.1.0/tests/unit/metrics/test_suite.py +87 -0
- strop_cli-0.1.0/tests/unit/metrics/test_text_and_rules.py +87 -0
- strop_cli-0.1.0/tests/unit/optimization/__init__.py +0 -0
- strop_cli-0.1.0/tests/unit/optimization/test_acceptance.py +150 -0
- strop_cli-0.1.0/tests/unit/optimization/test_generator.py +155 -0
- strop_cli-0.1.0/tests/unit/optimization/test_isolation.py +33 -0
- strop_cli-0.1.0/tests/unit/prompts/__init__.py +0 -0
- strop_cli-0.1.0/tests/unit/prompts/test_diff.py +64 -0
- strop_cli-0.1.0/tests/unit/prompts/test_export_formats.py +68 -0
- strop_cli-0.1.0/tests/unit/prompts/test_files.py +84 -0
- strop_cli-0.1.0/tests/unit/prompts/test_template.py +83 -0
- strop_cli-0.1.0/tests/unit/prompts/test_validation.py +58 -0
- strop_cli-0.1.0/tests/unit/services/__init__.py +0 -0
- strop_cli-0.1.0/tests/unit/services/test_evaluator.py +70 -0
- strop_cli-0.1.0/tests/unit/services/test_metric_setup.py +72 -0
- strop_cli-0.1.0/tests/unit/storage/__init__.py +0 -0
- strop_cli-0.1.0/tests/unit/storage/test_database.py +61 -0
- strop_cli-0.1.0/tests/unit/storage/test_datasets.py +63 -0
- strop_cli-0.1.0/tests/unit/storage/test_evaluations.py +113 -0
- strop_cli-0.1.0/tests/unit/storage/test_migrator.py +53 -0
- strop_cli-0.1.0/tests/unit/storage/test_prompts.py +74 -0
- strop_cli-0.1.0/tests/unit/storage/test_runs.py +115 -0
- strop_cli-0.1.0/tests/unit/test_docs.py +82 -0
- strop_cli-0.1.0/tests/unit/test_env.py +64 -0
- strop_cli-0.1.0/tests/unit/test_errors.py +34 -0
- strop_cli-0.1.0/tests/unit/test_log.py +48 -0
- strop_cli-0.1.0/tests/unit/test_package.py +7 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Local-only files
|
|
2
|
+
.claude
|
|
3
|
+
CLAUDE.md
|
|
4
|
+
|
|
5
|
+
# Python
|
|
6
|
+
__pycache__/
|
|
7
|
+
*.py[cod]
|
|
8
|
+
*.egg-info/
|
|
9
|
+
.eggs/
|
|
10
|
+
build/
|
|
11
|
+
dist/
|
|
12
|
+
|
|
13
|
+
# Virtual environments
|
|
14
|
+
.venv/
|
|
15
|
+
venv/
|
|
16
|
+
env/
|
|
17
|
+
|
|
18
|
+
# Tool caches and reports
|
|
19
|
+
.pytest_cache/
|
|
20
|
+
.mypy_cache/
|
|
21
|
+
.ruff_cache/
|
|
22
|
+
.coverage
|
|
23
|
+
.coverage.*
|
|
24
|
+
htmlcov/
|
|
25
|
+
coverage.xml
|
|
26
|
+
|
|
27
|
+
# Secrets
|
|
28
|
+
.env
|
|
29
|
+
.env.*
|
|
30
|
+
!.env.example
|
|
31
|
+
|
|
32
|
+
# Strop runtime data (created inside user projects)
|
|
33
|
+
.strop/
|
|
34
|
+
|
|
35
|
+
# Editors and OS
|
|
36
|
+
.vscode/
|
|
37
|
+
.idea/
|
|
38
|
+
.DS_Store
|
|
39
|
+
Thumbs.db
|
strop_cli-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Baran Yalcin
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
strop_cli-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: strop-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Command-line tool that improves LLM prompts by testing them against your own data.
|
|
5
|
+
Project-URL: Homepage, https://github.com/baranylcn/strop
|
|
6
|
+
Project-URL: Documentation, https://github.com/baranylcn/strop/tree/main/docs
|
|
7
|
+
Project-URL: Changelog, https://github.com/baranylcn/strop/blob/main/CHANGELOG.md
|
|
8
|
+
Project-URL: Issues, https://github.com/baranylcn/strop/issues
|
|
9
|
+
Author: Baran Yalcin
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: cli,evaluation,llm,prompt,prompt-optimization
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Typing :: Typed
|
|
22
|
+
Requires-Python: >=3.11
|
|
23
|
+
Requires-Dist: jsonschema>=4.23
|
|
24
|
+
Requires-Dist: litellm>=1.60
|
|
25
|
+
Requires-Dist: pydantic>=2.9
|
|
26
|
+
Requires-Dist: ruamel-yaml>=0.18
|
|
27
|
+
Requires-Dist: typer>=0.15
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# Strop
|
|
31
|
+
|
|
32
|
+
Strop is a command-line tool that improves LLM prompts by testing them against your own data. It runs your prompt, scores the outputs with a metric you define, asks a model to revise the prompt based on the failures, and keeps a revision only when the measured results get better.
|
|
33
|
+
|
|
34
|
+
## Status
|
|
35
|
+
|
|
36
|
+
Alpha. Version 0.1.0. Commands and the `strop.yaml` format may change between releases.
|
|
37
|
+
|
|
38
|
+
## What it does
|
|
39
|
+
|
|
40
|
+
You give Strop a prompt, a dataset, and a definition of success. Strop then repeats this loop, for as many turns as you allow:
|
|
41
|
+
|
|
42
|
+
1. Score the current prompt on a validation set.
|
|
43
|
+
2. Run it on a separate training set and collect the failures, grouped by error type.
|
|
44
|
+
3. Ask an optimizer model for revised prompts that address those failures. Each revision must cite the examples that motivated it.
|
|
45
|
+
4. Score every revision on the same validation rows. Keep one only if the primary metric improves and none of your constraints is broken.
|
|
46
|
+
5. Start the next turn from the best prompt.
|
|
47
|
+
|
|
48
|
+
When the loop ends, you run the final prompt once on a held-out test set. Strop prints a colored diff between your original prompt and the final one.
|
|
49
|
+
|
|
50
|
+
Everything is local. Prompt versions, results, and cached model responses are stored in a `.strop/` directory inside your project. Strop has no server and no account, and it sends nothing anywhere except the model providers you configure.
|
|
51
|
+
|
|
52
|
+
## Install
|
|
53
|
+
|
|
54
|
+
Strop needs Python 3.11 or newer.
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
pip install strop-cli
|
|
58
|
+
strop --version
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
The package is called `strop-cli` on PyPI because the name `strop` is reserved there. The command it installs is `strop`. To keep Strop out of your other Python environments, install it with `pipx install strop-cli` instead. To work on Strop itself, see [Development](#development).
|
|
62
|
+
|
|
63
|
+
Strop calls models through [LiteLLM](https://docs.litellm.ai/docs/providers), so you can use any provider and model it supports. Set the API key your provider expects as an environment variable, or put it in a `.env` file in your project directory. Strop has no default model; you name every model yourself.
|
|
64
|
+
|
|
65
|
+
## Quick start
|
|
66
|
+
|
|
67
|
+
The [repository](https://github.com/baranylcn/strop) includes a ready-to-run example in `examples/ticket_routing`: 150 synthetic support tickets in five categories, with a deliberately vague starting prompt. Clone the repository to get it.
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
cd examples/ticket_routing
|
|
71
|
+
|
|
72
|
+
strop init \
|
|
73
|
+
--name ticket-routing \
|
|
74
|
+
--task "Route a customer message to a support category" \
|
|
75
|
+
--prompt prompt.yaml \
|
|
76
|
+
--data data.csv --id-column ticket_id \
|
|
77
|
+
--input customer_message --expected category --metadata channel \
|
|
78
|
+
--output-type label --labels billing,refund,exchange,technical,other \
|
|
79
|
+
--metric macro_f1 --constraint "valid_output_rate>=0.98" \
|
|
80
|
+
--target-model <provider/model> --optimizer-model <provider/model>
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
`strop init` writes `strop.yaml` and a `.strop/` directory, and tells you which providers will receive your dataset rows. Nothing is sent yet. Next, check and split the data:
|
|
84
|
+
|
|
85
|
+
```console
|
|
86
|
+
$ strop data check
|
|
87
|
+
Dataset: data.csv
|
|
88
|
+
Records: 150, valid: 150, invalid: 0
|
|
89
|
+
Columns: ticket_id, customer_message, category, channel
|
|
90
|
+
|
|
91
|
+
$ strop data split
|
|
92
|
+
Created split 1 (seed 42) over 150 rows:
|
|
93
|
+
train 90
|
|
94
|
+
validation 30
|
|
95
|
+
test 30
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
The split is stored, so it never changes unless you ask for a new one. Now measure the starting prompt and start the loop:
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
strop baseline
|
|
102
|
+
strop optimize --max-turns 5
|
|
103
|
+
strop test --confirm
|
|
104
|
+
strop export
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
These commands call your models, so their output depends on which models you choose and is not shown here. `strop optimize` prints each turn as it goes and, at the end, the metric before and after, a table of every candidate and why it was kept or rejected, and the prompt diff. `strop export` writes the final prompt and a report to `exports/`.
|
|
108
|
+
|
|
109
|
+
## How success is measured
|
|
110
|
+
|
|
111
|
+
Strop cannot know what a correct output is. You define it, and Strop measures your definition.
|
|
112
|
+
|
|
113
|
+
| Way to define success | Use it when |
|
|
114
|
+
| --- | --- |
|
|
115
|
+
| Labeled data (`--expected`) | You have the right answer for each row: classification, extraction. |
|
|
116
|
+
| Rules (`--rule`) | You can check the output directly: length, required words, format. |
|
|
117
|
+
| A Python function (`--metric-file`) | You can compute a score in code. |
|
|
118
|
+
| A judge rubric (`--rubric-file`) | Quality needs judgment and there are no reference answers. |
|
|
119
|
+
|
|
120
|
+
Judge scores come from another language model, so they are estimates. Strop labels them as subjective and never mixes them with deterministic metrics. See [docs/metrics.md](https://github.com/baranylcn/strop/blob/main/docs/metrics.md) for every metric and `examples/support_replies` for rules, a custom metric, and judge rubrics together.
|
|
121
|
+
|
|
122
|
+
## How a revision is accepted
|
|
123
|
+
|
|
124
|
+
A candidate becomes the best prompt only if the primary metric improves by a minimum amount, every constraint you set still holds, the share of valid outputs does not drop, no class loses more than a set amount of F1, and the candidate wins on more validation examples than it loses. Every failed rule is recorded with the candidate.
|
|
125
|
+
|
|
126
|
+
The test partition is locked in code. Nothing the optimizer sees comes from it, and `strop test` refuses to run without `--confirm`, or a second time without `--allow-rerun`, because tuning a prompt after looking at the test result makes the result optimistic. [docs/how-it-works.md](https://github.com/baranylcn/strop/blob/main/docs/how-it-works.md) explains the rules in detail.
|
|
127
|
+
|
|
128
|
+
## Commands
|
|
129
|
+
|
|
130
|
+
| Command | Purpose |
|
|
131
|
+
| --- | --- |
|
|
132
|
+
| `strop init` | Create `strop.yaml`, a starter prompt, and the `.strop/` directory. |
|
|
133
|
+
| `strop data check` | Validate the dataset and list invalid rows with their row numbers. |
|
|
134
|
+
| `strop data split` | Create the train, validation, and test split. |
|
|
135
|
+
| `strop baseline` | Measure the initial prompt on the validation set. |
|
|
136
|
+
| `strop eval` | Score any prompt version on the train or validation set. Changes nothing. |
|
|
137
|
+
| `strop analyze` | Count and list how a prompt fails on the training data. |
|
|
138
|
+
| `strop optimize` | Run the optimization loop. Resumable. |
|
|
139
|
+
| `strop approve`, `strop reject` | Decide a candidate when running with `--approval manual`. |
|
|
140
|
+
| `strop test` | Run the final evaluation on the held-out test set. |
|
|
141
|
+
| `strop compare` | Run one prompt on several models and compare the results. |
|
|
142
|
+
| `strop diff`, `strop history` | Compare prompt versions and see every decision. |
|
|
143
|
+
| `strop prompt add`, `strop prompt show` | Save your own edit as a version, or print a version. |
|
|
144
|
+
| `strop rollback` | Make an earlier version the best prompt. Nothing is deleted. |
|
|
145
|
+
| `strop status` | Show the split, the best prompt, and recent evaluations. |
|
|
146
|
+
| `strop export` | Write the prompt, a JSON and Markdown report, and per-example results. |
|
|
147
|
+
| `strop clean` | Delete stored cache, outputs, or the whole `.strop/` directory. |
|
|
148
|
+
|
|
149
|
+
Run `strop <command> --help` for the options of each command. The commands never ask questions: every choice is a flag or a value in `strop.yaml`, so they work in scripts. Commands that print results accept `--json` for machine-readable output, and every command accepts `--dir` to name the project directory.
|
|
150
|
+
|
|
151
|
+
## Configuration
|
|
152
|
+
|
|
153
|
+
`strop init` writes `strop.yaml`. This is a minimal one:
|
|
154
|
+
|
|
155
|
+
```yaml
|
|
156
|
+
project:
|
|
157
|
+
name: ticket-routing
|
|
158
|
+
task_description: Route a customer message to a support category
|
|
159
|
+
dataset:
|
|
160
|
+
path: data.csv
|
|
161
|
+
columns: {customer_message: input, category: expected}
|
|
162
|
+
output:
|
|
163
|
+
type: label
|
|
164
|
+
labels: [billing, refund, exchange, technical, other]
|
|
165
|
+
models:
|
|
166
|
+
target: {model: <provider/model>}
|
|
167
|
+
optimizer: {model: <provider/model>}
|
|
168
|
+
metrics:
|
|
169
|
+
primary: macro_f1
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
The prompt itself is a separate file with chat messages and a list of the messages the optimizer may change:
|
|
173
|
+
|
|
174
|
+
```yaml
|
|
175
|
+
messages:
|
|
176
|
+
- role: system
|
|
177
|
+
content: |-
|
|
178
|
+
You route customer messages to a support team.
|
|
179
|
+
Reply with the category only.
|
|
180
|
+
- role: user
|
|
181
|
+
content: |-
|
|
182
|
+
Message: {{customer_message}}
|
|
183
|
+
optimize: [system]
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
The optimizer never touches messages outside `optimize`, never changes `{{variables}}`, and never changes text wrapped in `<strop:fixed>...</strop:fixed>`. Your file is not overwritten; revisions are stored as versions. The full reference is in [docs/configuration.md](https://github.com/baranylcn/strop/blob/main/docs/configuration.md).
|
|
187
|
+
|
|
188
|
+
## Models and providers
|
|
189
|
+
|
|
190
|
+
The roles are `target` (the model your prompt is for), `optimizer` (proposes revisions), `analyzer` (proposes failure patterns, defaults to the optimizer), and `judge` (scores rubrics). Each can be a different provider and model. Model names are LiteLLM names, such as `<provider>/<model>`. Pass `--optimizer-model` or `--analyzer-model` to `strop optimize` to change a role for one run, and use `strop compare --models a,b` to compare target models on the same rows.
|
|
191
|
+
|
|
192
|
+
Strop retries temporary provider errors with exponential backoff. A call that still fails is recorded as an error for that one example and does not stop the run.
|
|
193
|
+
|
|
194
|
+
## Data and privacy
|
|
195
|
+
|
|
196
|
+
- Dataset rows go to the providers of the models you configure. The target model receives every row it is run on. The optimizer and analyzer receive a bounded sample of failed train examples. The judge receives each example it scores. `strop init` lists the providers.
|
|
197
|
+
- `evidence_examples` and `evidence_chars` in `strop.yaml` limit how much of the failed examples is shown to the optimizer and analyzer.
|
|
198
|
+
- Dataset content is always passed to models as quoted data, and the meta-prompts tell the models not to follow instructions found inside it.
|
|
199
|
+
- API keys are read from the environment or `.env` and are never stored by Strop. Key-like strings are removed from logs and provider error messages.
|
|
200
|
+
- `.strop/` holds model outputs and cached responses. Keep it and `.env` out of version control. `strop clean` deletes what you choose.
|
|
201
|
+
- A `metric.py` file is your own code and runs as written, in a separate process. Strop never executes anything found in a dataset.
|
|
202
|
+
|
|
203
|
+
## Exit codes
|
|
204
|
+
|
|
205
|
+
| Code | Meaning |
|
|
206
|
+
| --- | --- |
|
|
207
|
+
| 0 | Success. |
|
|
208
|
+
| 1 | The command failed. |
|
|
209
|
+
| 2 | Usage or configuration error. |
|
|
210
|
+
| 10 | `strop optimize` paused and waits for `strop approve` or `strop reject`. |
|
|
211
|
+
| 11 | `strop optimize` stopped because a budget was reached. |
|
|
212
|
+
| 12 | The command was interrupted. Progress is saved. |
|
|
213
|
+
|
|
214
|
+
## Limitations
|
|
215
|
+
|
|
216
|
+
- A small validation set gives noisy comparisons. Strop warns below 30 validation rows and refuses to optimize below 10.
|
|
217
|
+
- The test set is only a fair estimate if you look at it once.
|
|
218
|
+
- Judge metrics are estimates, and a judge model can prefer its own style.
|
|
219
|
+
- The optimizer is a language model. It proposes revisions that fit the failures it is shown, and the validation rules decide whether they help. A revision that passes can still be tuned to the validation set, which is what the test set is for.
|
|
220
|
+
- Results depend on the models you use. A prompt tuned for one model may not transfer to another.
|
|
221
|
+
- Token and cost data come from the provider. When a provider reports none, Strop shows it as unavailable instead of estimating it.
|
|
222
|
+
- Strop optimizes single prompts, not agents or tool chains.
|
|
223
|
+
|
|
224
|
+
## Development
|
|
225
|
+
|
|
226
|
+
Strop uses [uv](https://docs.astral.sh/uv/).
|
|
227
|
+
|
|
228
|
+
```bash
|
|
229
|
+
git clone https://github.com/baranylcn/strop.git
|
|
230
|
+
cd strop
|
|
231
|
+
uv sync
|
|
232
|
+
uv run pytest
|
|
233
|
+
uv run ruff check .
|
|
234
|
+
uv run ruff format --check .
|
|
235
|
+
uv run mypy
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
The tests use scripted models and make no network calls. See [CONTRIBUTING.md](https://github.com/baranylcn/strop/blob/main/CONTRIBUTING.md) for the project layout and conventions.
|
|
239
|
+
|
|
240
|
+
## License
|
|
241
|
+
|
|
242
|
+
MIT. See [LICENSE](https://github.com/baranylcn/strop/blob/main/LICENSE).
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
# Strop
|
|
2
|
+
|
|
3
|
+
Strop is a command-line tool that improves LLM prompts by testing them against your own data. It runs your prompt, scores the outputs with a metric you define, asks a model to revise the prompt based on the failures, and keeps a revision only when the measured results get better.
|
|
4
|
+
|
|
5
|
+
## Status
|
|
6
|
+
|
|
7
|
+
Alpha. Version 0.1.0. Commands and the `strop.yaml` format may change between releases.
|
|
8
|
+
|
|
9
|
+
## What it does
|
|
10
|
+
|
|
11
|
+
You give Strop a prompt, a dataset, and a definition of success. Strop then repeats this loop, for as many turns as you allow:
|
|
12
|
+
|
|
13
|
+
1. Score the current prompt on a validation set.
|
|
14
|
+
2. Run it on a separate training set and collect the failures, grouped by error type.
|
|
15
|
+
3. Ask an optimizer model for revised prompts that address those failures. Each revision must cite the examples that motivated it.
|
|
16
|
+
4. Score every revision on the same validation rows. Keep one only if the primary metric improves and none of your constraints is broken.
|
|
17
|
+
5. Start the next turn from the best prompt.
|
|
18
|
+
|
|
19
|
+
When the loop ends, you run the final prompt once on a held-out test set. Strop prints a colored diff between your original prompt and the final one.
|
|
20
|
+
|
|
21
|
+
Everything is local. Prompt versions, results, and cached model responses are stored in a `.strop/` directory inside your project. Strop has no server and no account, and it sends nothing anywhere except the model providers you configure.
|
|
22
|
+
|
|
23
|
+
## Install
|
|
24
|
+
|
|
25
|
+
Strop needs Python 3.11 or newer.
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install strop-cli
|
|
29
|
+
strop --version
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
The package is called `strop-cli` on PyPI because the name `strop` is reserved there. The command it installs is `strop`. To keep Strop out of your other Python environments, install it with `pipx install strop-cli` instead. To work on Strop itself, see [Development](#development).
|
|
33
|
+
|
|
34
|
+
Strop calls models through [LiteLLM](https://docs.litellm.ai/docs/providers), so you can use any provider and model it supports. Set the API key your provider expects as an environment variable, or put it in a `.env` file in your project directory. Strop has no default model; you name every model yourself.
|
|
35
|
+
|
|
36
|
+
## Quick start
|
|
37
|
+
|
|
38
|
+
The [repository](https://github.com/baranylcn/strop) includes a ready-to-run example in `examples/ticket_routing`: 150 synthetic support tickets in five categories, with a deliberately vague starting prompt. Clone the repository to get it.
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
cd examples/ticket_routing
|
|
42
|
+
|
|
43
|
+
strop init \
|
|
44
|
+
--name ticket-routing \
|
|
45
|
+
--task "Route a customer message to a support category" \
|
|
46
|
+
--prompt prompt.yaml \
|
|
47
|
+
--data data.csv --id-column ticket_id \
|
|
48
|
+
--input customer_message --expected category --metadata channel \
|
|
49
|
+
--output-type label --labels billing,refund,exchange,technical,other \
|
|
50
|
+
--metric macro_f1 --constraint "valid_output_rate>=0.98" \
|
|
51
|
+
--target-model <provider/model> --optimizer-model <provider/model>
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
`strop init` writes `strop.yaml` and a `.strop/` directory, and tells you which providers will receive your dataset rows. Nothing is sent yet. Next, check and split the data:
|
|
55
|
+
|
|
56
|
+
```console
|
|
57
|
+
$ strop data check
|
|
58
|
+
Dataset: data.csv
|
|
59
|
+
Records: 150, valid: 150, invalid: 0
|
|
60
|
+
Columns: ticket_id, customer_message, category, channel
|
|
61
|
+
|
|
62
|
+
$ strop data split
|
|
63
|
+
Created split 1 (seed 42) over 150 rows:
|
|
64
|
+
train 90
|
|
65
|
+
validation 30
|
|
66
|
+
test 30
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
The split is stored, so it never changes unless you ask for a new one. Now measure the starting prompt and start the loop:
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
strop baseline
|
|
73
|
+
strop optimize --max-turns 5
|
|
74
|
+
strop test --confirm
|
|
75
|
+
strop export
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
These commands call your models, so their output depends on which models you choose and is not shown here. `strop optimize` prints each turn as it goes and, at the end, the metric before and after, a table of every candidate and why it was kept or rejected, and the prompt diff. `strop export` writes the final prompt and a report to `exports/`.
|
|
79
|
+
|
|
80
|
+
## How success is measured
|
|
81
|
+
|
|
82
|
+
Strop cannot know what a correct output is. You define it, and Strop measures your definition.
|
|
83
|
+
|
|
84
|
+
| Way to define success | Use it when |
|
|
85
|
+
| --- | --- |
|
|
86
|
+
| Labeled data (`--expected`) | You have the right answer for each row: classification, extraction. |
|
|
87
|
+
| Rules (`--rule`) | You can check the output directly: length, required words, format. |
|
|
88
|
+
| A Python function (`--metric-file`) | You can compute a score in code. |
|
|
89
|
+
| A judge rubric (`--rubric-file`) | Quality needs judgment and there are no reference answers. |
|
|
90
|
+
|
|
91
|
+
Judge scores come from another language model, so they are estimates. Strop labels them as subjective and never mixes them with deterministic metrics. See [docs/metrics.md](https://github.com/baranylcn/strop/blob/main/docs/metrics.md) for every metric and `examples/support_replies` for rules, a custom metric, and judge rubrics together.
|
|
92
|
+
|
|
93
|
+
## How a revision is accepted
|
|
94
|
+
|
|
95
|
+
A candidate becomes the best prompt only if the primary metric improves by a minimum amount, every constraint you set still holds, the share of valid outputs does not drop, no class loses more than a set amount of F1, and the candidate wins on more validation examples than it loses. Every failed rule is recorded with the candidate.
|
|
96
|
+
|
|
97
|
+
The test partition is locked in code. Nothing the optimizer sees comes from it, and `strop test` refuses to run without `--confirm`, or a second time without `--allow-rerun`, because tuning a prompt after looking at the test result makes the result optimistic. [docs/how-it-works.md](https://github.com/baranylcn/strop/blob/main/docs/how-it-works.md) explains the rules in detail.
|
|
98
|
+
|
|
99
|
+
## Commands
|
|
100
|
+
|
|
101
|
+
| Command | Purpose |
|
|
102
|
+
| --- | --- |
|
|
103
|
+
| `strop init` | Create `strop.yaml`, a starter prompt, and the `.strop/` directory. |
|
|
104
|
+
| `strop data check` | Validate the dataset and list invalid rows with their row numbers. |
|
|
105
|
+
| `strop data split` | Create the train, validation, and test split. |
|
|
106
|
+
| `strop baseline` | Measure the initial prompt on the validation set. |
|
|
107
|
+
| `strop eval` | Score any prompt version on the train or validation set. Changes nothing. |
|
|
108
|
+
| `strop analyze` | Count and list how a prompt fails on the training data. |
|
|
109
|
+
| `strop optimize` | Run the optimization loop. Resumable. |
|
|
110
|
+
| `strop approve`, `strop reject` | Decide a candidate when running with `--approval manual`. |
|
|
111
|
+
| `strop test` | Run the final evaluation on the held-out test set. |
|
|
112
|
+
| `strop compare` | Run one prompt on several models and compare the results. |
|
|
113
|
+
| `strop diff`, `strop history` | Compare prompt versions and see every decision. |
|
|
114
|
+
| `strop prompt add`, `strop prompt show` | Save your own edit as a version, or print a version. |
|
|
115
|
+
| `strop rollback` | Make an earlier version the best prompt. Nothing is deleted. |
|
|
116
|
+
| `strop status` | Show the split, the best prompt, and recent evaluations. |
|
|
117
|
+
| `strop export` | Write the prompt, a JSON and Markdown report, and per-example results. |
|
|
118
|
+
| `strop clean` | Delete stored cache, outputs, or the whole `.strop/` directory. |
|
|
119
|
+
|
|
120
|
+
Run `strop <command> --help` for the options of each command. The commands never ask questions: every choice is a flag or a value in `strop.yaml`, so they work in scripts. Commands that print results accept `--json` for machine-readable output, and every command accepts `--dir` to name the project directory.
|
|
121
|
+
|
|
122
|
+
## Configuration
|
|
123
|
+
|
|
124
|
+
`strop init` writes `strop.yaml`. This is a minimal one:
|
|
125
|
+
|
|
126
|
+
```yaml
|
|
127
|
+
project:
|
|
128
|
+
name: ticket-routing
|
|
129
|
+
task_description: Route a customer message to a support category
|
|
130
|
+
dataset:
|
|
131
|
+
path: data.csv
|
|
132
|
+
columns: {customer_message: input, category: expected}
|
|
133
|
+
output:
|
|
134
|
+
type: label
|
|
135
|
+
labels: [billing, refund, exchange, technical, other]
|
|
136
|
+
models:
|
|
137
|
+
target: {model: <provider/model>}
|
|
138
|
+
optimizer: {model: <provider/model>}
|
|
139
|
+
metrics:
|
|
140
|
+
primary: macro_f1
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
The prompt itself is a separate file with chat messages and a list of the messages the optimizer may change:
|
|
144
|
+
|
|
145
|
+
```yaml
|
|
146
|
+
messages:
|
|
147
|
+
- role: system
|
|
148
|
+
content: |-
|
|
149
|
+
You route customer messages to a support team.
|
|
150
|
+
Reply with the category only.
|
|
151
|
+
- role: user
|
|
152
|
+
content: |-
|
|
153
|
+
Message: {{customer_message}}
|
|
154
|
+
optimize: [system]
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
The optimizer never touches messages outside `optimize`, never changes `{{variables}}`, and never changes text wrapped in `<strop:fixed>...</strop:fixed>`. Your file is not overwritten; revisions are stored as versions. The full reference is in [docs/configuration.md](https://github.com/baranylcn/strop/blob/main/docs/configuration.md).
|
|
158
|
+
|
|
159
|
+
## Models and providers
|
|
160
|
+
|
|
161
|
+
The roles are `target` (the model your prompt is for), `optimizer` (proposes revisions), `analyzer` (proposes failure patterns, defaults to the optimizer), and `judge` (scores rubrics). Each can be a different provider and model. Model names are LiteLLM names, such as `<provider>/<model>`. Pass `--optimizer-model` or `--analyzer-model` to `strop optimize` to change a role for one run, and use `strop compare --models a,b` to compare target models on the same rows.
|
|
162
|
+
|
|
163
|
+
Strop retries temporary provider errors with exponential backoff. A call that still fails is recorded as an error for that one example and does not stop the run.
|
|
164
|
+
|
|
165
|
+
## Data and privacy
|
|
166
|
+
|
|
167
|
+
- Dataset rows go to the providers of the models you configure. The target model receives every row it is run on. The optimizer and analyzer receive a bounded sample of failed train examples. The judge receives each example it scores. `strop init` lists the providers.
|
|
168
|
+
- `evidence_examples` and `evidence_chars` in `strop.yaml` limit how much of the failed examples is shown to the optimizer and analyzer.
|
|
169
|
+
- Dataset content is always passed to models as quoted data, and the meta-prompts tell the models not to follow instructions found inside it.
|
|
170
|
+
- API keys are read from the environment or `.env` and are never stored by Strop. Key-like strings are removed from logs and provider error messages.
|
|
171
|
+
- `.strop/` holds model outputs and cached responses. Keep it and `.env` out of version control. `strop clean` deletes what you choose.
|
|
172
|
+
- A `metric.py` file is your own code and runs as written, in a separate process. Strop never executes anything found in a dataset.
|
|
173
|
+
|
|
174
|
+
## Exit codes
|
|
175
|
+
|
|
176
|
+
| Code | Meaning |
|
|
177
|
+
| --- | --- |
|
|
178
|
+
| 0 | Success. |
|
|
179
|
+
| 1 | The command failed. |
|
|
180
|
+
| 2 | Usage or configuration error. |
|
|
181
|
+
| 10 | `strop optimize` paused and waits for `strop approve` or `strop reject`. |
|
|
182
|
+
| 11 | `strop optimize` stopped because a budget was reached. |
|
|
183
|
+
| 12 | The command was interrupted. Progress is saved. |
|
|
184
|
+
|
|
185
|
+
## Limitations
|
|
186
|
+
|
|
187
|
+
- A small validation set gives noisy comparisons. Strop warns below 30 validation rows and refuses to optimize below 10.
|
|
188
|
+
- The test set is only a fair estimate if you look at it once.
|
|
189
|
+
- Judge metrics are estimates, and a judge model can prefer its own style.
|
|
190
|
+
- The optimizer is a language model. It proposes revisions that fit the failures it is shown, and the validation rules decide whether they help. A revision that passes can still be tuned to the validation set, which is what the test set is for.
|
|
191
|
+
- Results depend on the models you use. A prompt tuned for one model may not transfer to another.
|
|
192
|
+
- Token and cost data come from the provider. When a provider reports none, Strop shows it as unavailable instead of estimating it.
|
|
193
|
+
- Strop optimizes single prompts, not agents or tool chains.
|
|
194
|
+
|
|
195
|
+
## Development
|
|
196
|
+
|
|
197
|
+
Strop uses [uv](https://docs.astral.sh/uv/).
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
git clone https://github.com/baranylcn/strop.git
|
|
201
|
+
cd strop
|
|
202
|
+
uv sync
|
|
203
|
+
uv run pytest
|
|
204
|
+
uv run ruff check .
|
|
205
|
+
uv run ruff format --check .
|
|
206
|
+
uv run mypy
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
The tests use scripted models and make no network calls. See [CONTRIBUTING.md](https://github.com/baranylcn/strop/blob/main/CONTRIBUTING.md) for the project layout and conventions.
|
|
210
|
+
|
|
211
|
+
## License
|
|
212
|
+
|
|
213
|
+
MIT. See [LICENSE](https://github.com/baranylcn/strop/blob/main/LICENSE).
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "strop-cli"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Command-line tool that improves LLM prompts by testing them against your own data."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Baran Yalcin" }]
|
|
14
|
+
keywords = ["llm", "prompt", "prompt-optimization", "evaluation", "cli"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Operating System :: OS Independent",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Typing :: Typed",
|
|
25
|
+
]
|
|
26
|
+
dependencies = [
|
|
27
|
+
"jsonschema>=4.23",
|
|
28
|
+
"litellm>=1.60",
|
|
29
|
+
"pydantic>=2.9",
|
|
30
|
+
"ruamel-yaml>=0.18",
|
|
31
|
+
"typer>=0.15",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[project.scripts]
|
|
35
|
+
strop = "strop.cli:main"
|
|
36
|
+
|
|
37
|
+
[project.urls]
|
|
38
|
+
Homepage = "https://github.com/baranylcn/strop"
|
|
39
|
+
Documentation = "https://github.com/baranylcn/strop/tree/main/docs"
|
|
40
|
+
Changelog = "https://github.com/baranylcn/strop/blob/main/CHANGELOG.md"
|
|
41
|
+
Issues = "https://github.com/baranylcn/strop/issues"
|
|
42
|
+
|
|
43
|
+
[dependency-groups]
|
|
44
|
+
dev = [
|
|
45
|
+
"mypy>=1.13",
|
|
46
|
+
"pytest>=8.3",
|
|
47
|
+
"pytest-asyncio>=0.24",
|
|
48
|
+
"ruff>=0.8",
|
|
49
|
+
"types-jsonschema>=4.26.0.20261006",
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
[tool.hatch.version]
|
|
53
|
+
path = "src/strop/__init__.py"
|
|
54
|
+
|
|
55
|
+
[tool.hatch.build.targets.wheel]
|
|
56
|
+
packages = ["src/strop"]
|
|
57
|
+
|
|
58
|
+
[tool.hatch.build.targets.sdist]
|
|
59
|
+
include = ["/src", "/tests", "/README.md", "/LICENSE", "/pyproject.toml"]
|
|
60
|
+
|
|
61
|
+
[tool.ruff]
|
|
62
|
+
line-length = 100
|
|
63
|
+
target-version = "py311"
|
|
64
|
+
src = ["src", "tests"]
|
|
65
|
+
|
|
66
|
+
[tool.ruff.format]
|
|
67
|
+
line-ending = "lf"
|
|
68
|
+
|
|
69
|
+
[tool.ruff.lint]
|
|
70
|
+
select = ["E", "F", "W", "I", "N", "UP", "B", "C4", "SIM", "RUF", "PT", "ASYNC", "PL"]
|
|
71
|
+
ignore = ["PLR2004"]
|
|
72
|
+
|
|
73
|
+
[tool.ruff.lint.per-file-ignores]
|
|
74
|
+
"src/strop/cli/*" = ["PLR0913", "PLR0917"]
|
|
75
|
+
|
|
76
|
+
[tool.ruff.lint.pylint]
|
|
77
|
+
max-args = 7
|
|
78
|
+
max-statements = 40
|
|
79
|
+
|
|
80
|
+
[tool.mypy]
|
|
81
|
+
python_version = "3.11"
|
|
82
|
+
strict = true
|
|
83
|
+
files = ["src", "tests"]
|
|
84
|
+
plugins = ["pydantic.mypy"]
|
|
85
|
+
|
|
86
|
+
[tool.pytest.ini_options]
|
|
87
|
+
testpaths = ["tests"]
|
|
88
|
+
asyncio_mode = "auto"
|
|
89
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
90
|
+
pythonpath = ["."]
|
|
91
|
+
addopts = "-ra --strict-markers --strict-config --import-mode=importlib"
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Failure analysis: deterministic facts first, LLM-proposed patterns as hypotheses."""
|
|
2
|
+
|
|
3
|
+
from strop.analysis.classify import classify_outcome
|
|
4
|
+
from strop.analysis.evidence import evidence_payload, select_evidence
|
|
5
|
+
from strop.analysis.facts import AnalysisFacts, FailedExample, build_facts
|
|
6
|
+
from strop.analysis.patterns import Pattern, find_patterns
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"AnalysisFacts",
|
|
10
|
+
"FailedExample",
|
|
11
|
+
"Pattern",
|
|
12
|
+
"build_facts",
|
|
13
|
+
"classify_outcome",
|
|
14
|
+
"evidence_payload",
|
|
15
|
+
"find_patterns",
|
|
16
|
+
"select_evidence",
|
|
17
|
+
]
|