strop-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. strop/__init__.py +5 -0
  2. strop/__main__.py +5 -0
  3. strop/analysis/__init__.py +17 -0
  4. strop/analysis/classify.py +52 -0
  5. strop/analysis/evidence.py +52 -0
  6. strop/analysis/facts.py +74 -0
  7. strop/analysis/patterns.py +90 -0
  8. strop/cli/__init__.py +5 -0
  9. strop/cli/analyze.py +93 -0
  10. strop/cli/app.py +82 -0
  11. strop/cli/data.py +111 -0
  12. strop/cli/evaluate.py +145 -0
  13. strop/cli/export.py +74 -0
  14. strop/cli/final.py +106 -0
  15. strop/cli/init.py +131 -0
  16. strop/cli/optimize.py +156 -0
  17. strop/cli/options.py +20 -0
  18. strop/cli/progress.py +18 -0
  19. strop/cli/prompts.py +110 -0
  20. strop/cli/status.py +68 -0
  21. strop/cli/versions.py +123 -0
  22. strop/config/__init__.py +16 -0
  23. strop/config/base.py +9 -0
  24. strop/config/constraints.py +33 -0
  25. strop/config/dataset.py +63 -0
  26. strop/config/enums.py +52 -0
  27. strop/config/io.py +52 -0
  28. strop/config/limits.py +19 -0
  29. strop/config/llm.py +42 -0
  30. strop/config/metrics.py +86 -0
  31. strop/config/optimization.py +23 -0
  32. strop/config/output.py +31 -0
  33. strop/config/overrides.py +22 -0
  34. strop/config/root.py +36 -0
  35. strop/config/rules.py +67 -0
  36. strop/config/split.py +23 -0
  37. strop/data/__init__.py +58 -0
  38. strop/data/loaders.py +85 -0
  39. strop/data/models.py +44 -0
  40. strop/data/rows.py +99 -0
  41. strop/data/splitting.py +103 -0
  42. strop/env.py +44 -0
  43. strop/errors.py +59 -0
  44. strop/evaluation/__init__.py +1 -0
  45. strop/evaluation/budget.py +67 -0
  46. strop/evaluation/budgeted.py +23 -0
  47. strop/evaluation/examples.py +49 -0
  48. strop/evaluation/models.py +105 -0
  49. strop/evaluation/parsing.py +96 -0
  50. strop/evaluation/runner.py +107 -0
  51. strop/formatting.py +14 -0
  52. strop/llm/__init__.py +7 -0
  53. strop/llm/caching.py +28 -0
  54. strop/llm/client.py +15 -0
  55. strop/llm/factory.py +14 -0
  56. strop/llm/litellm_client.py +90 -0
  57. strop/llm/mock.py +43 -0
  58. strop/llm/retry.py +44 -0
  59. strop/llm/structured.py +52 -0
  60. strop/llm/types.py +59 -0
  61. strop/log.py +49 -0
  62. strop/metrics/__init__.py +1 -0
  63. strop/metrics/basic.py +30 -0
  64. strop/metrics/classification.py +164 -0
  65. strop/metrics/judge.py +130 -0
  66. strop/metrics/python_metric.py +101 -0
  67. strop/metrics/python_runner.py +65 -0
  68. strop/metrics/registry.py +115 -0
  69. strop/metrics/rules.py +55 -0
  70. strop/metrics/structured.py +109 -0
  71. strop/metrics/suite.py +109 -0
  72. strop/metrics/text.py +55 -0
  73. strop/metrics/types.py +106 -0
  74. strop/optimization/__init__.py +25 -0
  75. strop/optimization/acceptance.py +170 -0
  76. strop/optimization/generator.py +92 -0
  77. strop/optimization/meta_prompt.py +101 -0
  78. strop/optimization/types.py +69 -0
  79. strop/prompts/__init__.py +18 -0
  80. strop/prompts/diff.py +139 -0
  81. strop/prompts/files.py +92 -0
  82. strop/prompts/models.py +51 -0
  83. strop/prompts/render.py +16 -0
  84. strop/prompts/template.py +45 -0
  85. strop/prompts/validation.py +31 -0
  86. strop/prompts/versions.py +34 -0
  87. strop/py.typed +0 -0
  88. strop/reporting/__init__.py +6 -0
  89. strop/reporting/console.py +20 -0
  90. strop/reporting/diff_view.py +102 -0
  91. strop/reporting/evaluation_view.py +141 -0
  92. strop/reporting/final_view.py +109 -0
  93. strop/reporting/prompt_export.py +58 -0
  94. strop/reporting/report_markdown.py +157 -0
  95. strop/reporting/run_view.py +118 -0
  96. strop/services/__init__.py +1 -0
  97. strop/services/analysis.py +124 -0
  98. strop/services/clean.py +38 -0
  99. strop/services/compare.py +60 -0
  100. strop/services/context.py +70 -0
  101. strop/services/data.py +65 -0
  102. strop/services/evaluate.py +105 -0
  103. strop/services/evaluator.py +169 -0
  104. strop/services/export.py +90 -0
  105. strop/services/final_test.py +69 -0
  106. strop/services/metrics.py +89 -0
  107. strop/services/optimize.py +187 -0
  108. strop/services/partitions.py +46 -0
  109. strop/services/project.py +55 -0
  110. strop/services/prompts.py +76 -0
  111. strop/services/report.py +194 -0
  112. strop/services/review.py +43 -0
  113. strop/services/scaffold.py +79 -0
  114. strop/services/turn.py +234 -0
  115. strop/storage/__init__.py +6 -0
  116. strop/storage/cache.py +45 -0
  117. strop/storage/database.py +45 -0
  118. strop/storage/datasets.py +129 -0
  119. strop/storage/evaluations.py +160 -0
  120. strop/storage/migrations/001_initial.sql +5 -0
  121. strop/storage/migrations/002_datasets.sql +27 -0
  122. strop/storage/migrations/003_prompt_versions.sql +22 -0
  123. strop/storage/migrations/004_llm_cache.sql +6 -0
  124. strop/storage/migrations/005_evaluations.sql +37 -0
  125. strop/storage/migrations/006_runs.sql +52 -0
  126. strop/storage/migrations/007_example_feedback.sql +2 -0
  127. strop/storage/migrations/__init__.py +5 -0
  128. strop/storage/migrator.py +91 -0
  129. strop/storage/paths.py +30 -0
  130. strop/storage/prompts.py +96 -0
  131. strop/storage/run_models.py +75 -0
  132. strop/storage/runs.py +263 -0
  133. strop_cli-0.1.0.dist-info/METADATA +242 -0
  134. strop_cli-0.1.0.dist-info/RECORD +137 -0
  135. strop_cli-0.1.0.dist-info/WHEEL +4 -0
  136. strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
  137. strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
strop/errors.py ADDED
@@ -0,0 +1,59 @@
1
+ """Exception hierarchy and process exit codes.
2
+
3
+ The command-line layer is the only place that turns an exception into an exit
4
+ code. Library code raises a specific ``StropError`` subclass and stays unaware
5
+ of how the process ends.
6
+ """
7
+
8
+ from enum import IntEnum
9
+
10
+
11
+ class ExitCode(IntEnum):
12
+ """Process exit codes documented in the CLI help."""
13
+
14
+ SUCCESS = 0
15
+ FAILURE = 1
16
+ USAGE = 2
17
+ PAUSED = 10
18
+ BUDGET = 11
19
+ INTERRUPTED = 12
20
+
21
+
22
+ class StropError(Exception):
23
+ """Base class for all expected, user-facing errors."""
24
+
25
+ exit_code: ExitCode = ExitCode.FAILURE
26
+
27
+
28
+ class ConfigError(StropError):
29
+ """The project configuration or a command-line value is missing or invalid."""
30
+
31
+ exit_code = ExitCode.USAGE
32
+
33
+
34
+ class DataError(StropError):
35
+ """A dataset file cannot be read or fails validation."""
36
+
37
+ exit_code = ExitCode.USAGE
38
+
39
+
40
+ class StorageError(StropError):
41
+ """The local project database or its files cannot be used."""
42
+
43
+
44
+ class LLMError(StropError):
45
+ """A model provider call failed after retries."""
46
+
47
+
48
+ class TransientLLMError(LLMError):
49
+ """A temporary provider failure (rate limit, timeout, outage) that may succeed on retry."""
50
+
51
+
52
+ class InvalidModelReplyError(LLMError):
53
+ """The model kept returning replies that do not match the requested JSON schema."""
54
+
55
+
56
+ class BudgetExceededError(StropError):
57
+ """A configured call, cost, or time budget was reached."""
58
+
59
+ exit_code = ExitCode.BUDGET
@@ -0,0 +1 @@
1
+ """Running prompts over data and scoring the outputs."""
@@ -0,0 +1,67 @@
1
+ """Tracking calls, cost and time against the configured budgets."""
2
+
3
+ import time
4
+ from collections.abc import Callable
5
+ from dataclasses import dataclass, field
6
+
7
+ from strop.config.limits import BudgetsConfig
8
+ from strop.errors import BudgetExceededError
9
+ from strop.llm.types import LLMResponse
10
+
11
+
12
+ @dataclass
13
+ class BudgetTracker:
14
+ """Counts fresh model calls and known cost, and raises once a limit is reached.
15
+
16
+ Cached responses are free and are not counted. A cost limit can only be enforced
17
+ for calls whose provider reported a cost; ``cost_unknown_calls`` shows the rest.
18
+ """
19
+
20
+ config: BudgetsConfig
21
+ clock: Callable[[], float] = time.monotonic
22
+ calls: int = 0
23
+ cost_usd: float = 0.0
24
+ cost_unknown_calls: int = 0
25
+ _start: float = field(init=False)
26
+
27
+ def __post_init__(self) -> None:
28
+ self._start = self.clock()
29
+
30
+ @property
31
+ def elapsed_minutes(self) -> float:
32
+ return (self.clock() - self._start) / 60
33
+
34
+ def record(self, response: LLMResponse) -> None:
35
+ if response.cached:
36
+ return
37
+ self.calls += 1
38
+ if response.usage.cost_usd is None:
39
+ self.cost_unknown_calls += 1
40
+ else:
41
+ self.cost_usd += response.usage.cost_usd
42
+
43
+ def record_failed_call(self) -> None:
44
+ self.calls += 1
45
+ self.cost_unknown_calls += 1
46
+
47
+ def check(self) -> None:
48
+ """Raise ``BudgetExceededError`` if any configured limit has been reached."""
49
+ limits = self.config
50
+ if limits.max_total_model_calls is not None and self.calls >= limits.max_total_model_calls:
51
+ raise BudgetExceededError(f"model call budget reached ({self.calls} calls)")
52
+ if limits.max_total_cost_usd is not None and self.cost_usd >= limits.max_total_cost_usd:
53
+ raise BudgetExceededError(f"cost budget reached (${self.cost_usd:.4f})")
54
+ if (
55
+ limits.max_wall_time_minutes is not None
56
+ and self.elapsed_minutes >= limits.max_wall_time_minutes
57
+ ):
58
+ raise BudgetExceededError(f"time budget reached ({self.elapsed_minutes:.1f} minutes)")
59
+
60
+ def warnings(self) -> list[str]:
61
+ """Caveats about limits that cannot be fully enforced."""
62
+ if self.config.max_total_cost_usd is not None and self.cost_unknown_calls:
63
+ return [
64
+ f"{self.cost_unknown_calls} call(s) reported no cost, so the cost budget "
65
+ "only covers the calls that did"
66
+ ]
67
+ return []
@@ -0,0 +1,23 @@
1
+ """A client wrapper that counts every call against the run budget."""
2
+
3
+ from strop.evaluation.budget import BudgetTracker
4
+ from strop.llm.client import LLMClient
5
+ from strop.llm.types import LLMRequest, LLMResponse
6
+
7
+
8
+ class BudgetedClient:
9
+ """Checks the budget before each call and records the call afterwards.
10
+
11
+ Used for the optimizer, analyzer and judge. Target-model calls are counted by
12
+ ``PromptRunner`` instead, so they must not go through this wrapper as well.
13
+ """
14
+
15
+ def __init__(self, inner: LLMClient, budget: BudgetTracker) -> None:
16
+ self._inner = inner
17
+ self._budget = budget
18
+
19
+ async def complete(self, request: LLMRequest) -> LLMResponse:
20
+ self._budget.check()
21
+ response = await self._inner.complete(request)
22
+ self._budget.record(response)
23
+ return response
@@ -0,0 +1,49 @@
1
+ """Joining stored outcomes with their dataset rows to form scorable examples."""
2
+
3
+ from collections.abc import Mapping, Sequence
4
+ from dataclasses import replace
5
+
6
+ from strop.data.models import Row
7
+ from strop.evaluation.models import ExampleOutcome
8
+ from strop.metrics.suite import MetricSuite
9
+ from strop.metrics.types import Example
10
+
11
+
12
+ def to_examples(rows: Sequence[Row], outcomes: Sequence[ExampleOutcome]) -> list[Example]:
13
+ """Pair each outcome with the inputs of its row."""
14
+ by_id: Mapping[str, Row] = {row.id: row for row in rows}
15
+ return [
16
+ Example(
17
+ row_id=o.row_id,
18
+ inputs=by_id[o.row_id].inputs,
19
+ expected=o.expected,
20
+ raw_output=o.raw_output,
21
+ value=o.value,
22
+ parsed=o.parsed,
23
+ error=o.error,
24
+ )
25
+ for o in outcomes
26
+ ]
27
+
28
+
29
+ def with_scores(
30
+ outcomes: Sequence[ExampleOutcome], examples: Sequence[Example], suite: MetricSuite
31
+ ) -> list[ExampleOutcome]:
32
+ """Attach each example's per-metric scores and metric feedback to its outcome."""
33
+ return [
34
+ replace(
35
+ outcome,
36
+ scores={name: suite.metric(name).per_example(example) for name in suite.names},
37
+ feedback=_feedback(example, suite),
38
+ )
39
+ for outcome, example in zip(outcomes, examples, strict=True)
40
+ ]
41
+
42
+
43
+ def _feedback(example: Example, suite: MetricSuite) -> str | None:
44
+ notes = []
45
+ for name in suite.names:
46
+ text = suite.metric(name).feedback(example)
47
+ if text:
48
+ notes.append(f"{name}: {text}")
49
+ return "; ".join(notes) or None
@@ -0,0 +1,105 @@
1
+ """Evaluation result types."""
2
+
3
+ from collections.abc import Sequence
4
+ from dataclasses import dataclass, field
5
+ from typing import Any
6
+
7
+ from strop.data.models import Partition
8
+ from strop.metrics.suite import MetricReport
9
+
10
+
11
+ @dataclass(frozen=True, slots=True)
12
+ class ExampleOutcome:
13
+ """The stored result for one row: the model output, its parse, and its per-metric scores."""
14
+
15
+ row_id: str
16
+ expected: str | None
17
+ raw_output: str
18
+ value: str | None
19
+ parsed: Any = None
20
+ error: str | None = None
21
+ finish_reason: str | None = None
22
+ latency_s: float = 0.0
23
+ prompt_tokens: int | None = None
24
+ completion_tokens: int | None = None
25
+ total_tokens: int | None = None
26
+ cost_usd: float | None = None
27
+ cached: bool = False
28
+ detail: str | None = None
29
+ feedback: str | None = None
30
+ scores: dict[str, float | None] = field(default_factory=dict)
31
+
32
+ @property
33
+ def failed(self) -> bool:
34
+ return self.error is not None
35
+
36
+
37
+ @dataclass(frozen=True, slots=True)
38
+ class UsageSummary:
39
+ """Calls, tokens, cost and latency. ``None`` means the provider reported nothing."""
40
+
41
+ calls: int = 0
42
+ cached_calls: int = 0
43
+ prompt_tokens: int | None = None
44
+ completion_tokens: int | None = None
45
+ total_tokens: int | None = None
46
+ cost_usd: float | None = None
47
+ cost_known_calls: int = 0
48
+ mean_latency_s: float | None = None
49
+
50
+ @property
51
+ def cost_complete(self) -> bool:
52
+ """True if every non-cached call reported a cost."""
53
+ return self.cost_known_calls == self.calls - self.cached_calls
54
+
55
+
56
+ def _sum_known(values: Sequence[int | float | None]) -> Any:
57
+ known = [v for v in values if v is not None]
58
+ return sum(known) if known else None
59
+
60
+
61
+ def summarize_usage(outcomes: Sequence[ExampleOutcome]) -> UsageSummary:
62
+ """Totals over ``outcomes``. Cached answers add no tokens or cost and no new call."""
63
+ fresh = [o for o in outcomes if not o.cached]
64
+ latencies = [o.latency_s for o in fresh if o.latency_s > 0]
65
+ return UsageSummary(
66
+ calls=len(outcomes),
67
+ cached_calls=len(outcomes) - len(fresh),
68
+ prompt_tokens=_sum_known([o.prompt_tokens for o in fresh]),
69
+ completion_tokens=_sum_known([o.completion_tokens for o in fresh]),
70
+ total_tokens=_sum_known([o.total_tokens for o in fresh]),
71
+ cost_usd=_sum_known([o.cost_usd for o in fresh]),
72
+ cost_known_calls=sum(o.cost_usd is not None for o in fresh),
73
+ mean_latency_s=sum(latencies) / len(latencies) if latencies else None,
74
+ )
75
+
76
+
77
+ @dataclass(frozen=True)
78
+ class EvaluationRecord:
79
+ """A stored evaluation of one prompt version on one partition."""
80
+
81
+ id: int
82
+ prompt_number: int
83
+ split_id: int
84
+ partition: Partition
85
+ kind: str
86
+ report: MetricReport
87
+ usage: UsageSummary
88
+ sample_count: int
89
+ error_count: int
90
+ created_at: str
91
+ details: dict[str, Any] = field(default_factory=dict)
92
+
93
+
94
+ @dataclass(frozen=True)
95
+ class NewEvaluation:
96
+ """Everything needed to store an evaluation."""
97
+
98
+ prompt_number: int
99
+ split_id: int
100
+ partition: Partition
101
+ kind: str
102
+ report: MetricReport
103
+ outcomes: Sequence[ExampleOutcome]
104
+ details: dict[str, Any] = field(default_factory=dict)
105
+ run_id: int | None = None
@@ -0,0 +1,96 @@
1
+ """Turning raw model text into a value that metrics can score."""
2
+
3
+ import json
4
+ import re
5
+ from dataclasses import dataclass
6
+ from pathlib import Path
7
+ from typing import Any
8
+
9
+ from jsonschema import Draft202012Validator
10
+ from jsonschema.exceptions import SchemaError
11
+
12
+ from strop.config.enums import OutputType
13
+ from strop.config.output import OutputConfig
14
+ from strop.errors import ConfigError
15
+ from strop.metrics.types import ErrorCode
16
+
17
+ _FENCE = re.compile(r"^```(?:json)?\s*(.*?)\s*```$", re.DOTALL | re.IGNORECASE)
18
+ _QUOTES = "`'\"" + "".join(chr(code) for code in (0x201C, 0x201D, 0x2018, 0x2019))
19
+
20
+
21
+ @dataclass(frozen=True, slots=True)
22
+ class ParsedOutput:
23
+ value: str | None
24
+ parsed: Any = None
25
+ error: ErrorCode | None = None
26
+
27
+
28
+ def canonical_json(value: Any) -> str:
29
+ return json.dumps(value, sort_keys=True, ensure_ascii=False)
30
+
31
+
32
+ def canonical_expected(text: str, output_type: OutputType) -> str:
33
+ """Normalize an expected value the same way outputs are normalized."""
34
+ stripped = text.strip()
35
+ if output_type is OutputType.JSON:
36
+ return canonical_json(json.loads(stripped))
37
+ return stripped
38
+
39
+
40
+ def load_output_schema(root: Path, output: OutputConfig) -> dict[str, Any] | None:
41
+ """Read and check the JSON schema named in the output contract, if any."""
42
+ if output.schema_file is None:
43
+ return None
44
+ path = root / output.schema_file
45
+ try:
46
+ schema = json.loads(path.read_text(encoding="utf-8"))
47
+ Draft202012Validator.check_schema(schema)
48
+ except FileNotFoundError:
49
+ raise ConfigError(f"output schema file not found: {path}") from None
50
+ except (ValueError, SchemaError) as exc:
51
+ raise ConfigError(f"output schema {path.name} is not a valid JSON schema: {exc}") from exc
52
+ return schema # type: ignore[no-any-return]
53
+
54
+
55
+ def parse_output(
56
+ raw: str, finish_reason: str | None, output: OutputConfig, schema: dict[str, Any] | None = None
57
+ ) -> ParsedOutput:
58
+ """Normalize ``raw`` under the output contract; failures carry an error code."""
59
+ text = raw.strip()
60
+ if not text:
61
+ return ParsedOutput(None, error=ErrorCode.EMPTY_OUTPUT)
62
+ failure = ErrorCode.TRUNCATED if finish_reason == "length" else ErrorCode.INVALID_FORMAT
63
+ if output.type is OutputType.LABEL:
64
+ label = _match_label(text, output)
65
+ return ParsedOutput(label) if label else ParsedOutput(None, error=failure)
66
+ if output.type is OutputType.JSON:
67
+ return _parse_json(text, schema, failure)
68
+ if finish_reason == "length":
69
+ return ParsedOutput(None, error=ErrorCode.TRUNCATED)
70
+ return ParsedOutput(text)
71
+
72
+
73
+ def _match_label(text: str, output: OutputConfig) -> str | None:
74
+ by_name = {label.casefold(): label for label in output.labels}
75
+ candidate = text.strip(_QUOTES + " \t\r\n").rstrip(".!").strip(_QUOTES + " ")
76
+ if candidate.casefold() in by_name:
77
+ return by_name[candidate.casefold()]
78
+ if output.match != "contains":
79
+ return None
80
+ found = [
81
+ label
82
+ for key, label in by_name.items()
83
+ if re.search(rf"(?<!\w){re.escape(key)}(?!\w)", text.casefold())
84
+ ]
85
+ return found[0] if len(found) == 1 else None
86
+
87
+
88
+ def _parse_json(text: str, schema: dict[str, Any] | None, failure: ErrorCode) -> ParsedOutput:
89
+ fenced = _FENCE.match(text)
90
+ try:
91
+ parsed = json.loads(fenced.group(1) if fenced else text)
92
+ except ValueError:
93
+ return ParsedOutput(None, error=failure)
94
+ if schema is not None and next(Draft202012Validator(schema).iter_errors(parsed), None):
95
+ return ParsedOutput(None, parsed, ErrorCode.SCHEMA_VIOLATION)
96
+ return ParsedOutput(canonical_json(parsed), parsed)
@@ -0,0 +1,107 @@
1
+ """Running a prompt over rows with bounded concurrency."""
2
+
3
+ import asyncio
4
+ from collections.abc import Callable, Sequence
5
+ from typing import Any
6
+
7
+ from strop.config.limits import ExecutionConfig
8
+ from strop.config.llm import ModelSpec
9
+ from strop.config.output import OutputConfig
10
+ from strop.data.models import Row
11
+ from strop.errors import LLMError
12
+ from strop.evaluation.budget import BudgetTracker
13
+ from strop.evaluation.models import ExampleOutcome
14
+ from strop.evaluation.parsing import canonical_expected, parse_output
15
+ from strop.llm.client import LLMClient
16
+ from strop.llm.types import LLMRequest, LLMResponse
17
+ from strop.log import redact
18
+ from strop.metrics.types import ErrorCode
19
+ from strop.prompts import PromptTemplate, render_messages
20
+
21
+ Progress = Callable[[int, int], None]
22
+
23
+
24
+ class PromptRunner:
25
+ """Sends one request per row and turns each reply into an ``ExampleOutcome``."""
26
+
27
+ def __init__(
28
+ self,
29
+ client: LLMClient,
30
+ spec: ModelSpec,
31
+ output: OutputConfig,
32
+ schema: dict[str, Any] | None,
33
+ execution: ExecutionConfig,
34
+ budget: BudgetTracker | None = None,
35
+ ) -> None:
36
+ self._client = client
37
+ self._spec = spec
38
+ self._output = output
39
+ self._schema = schema
40
+ self._execution = execution
41
+ self._budget = budget
42
+
43
+ async def run(
44
+ self,
45
+ template: PromptTemplate,
46
+ rows: Sequence[Row],
47
+ progress: Progress | None = None,
48
+ ) -> list[ExampleOutcome]:
49
+ """Evaluate ``rows`` in batches, checking the budget between batches."""
50
+ semaphore = asyncio.Semaphore(self._execution.max_concurrency)
51
+ outcomes: list[ExampleOutcome] = []
52
+ for start in range(0, len(rows), self._execution.batch_size):
53
+ if self._budget is not None:
54
+ self._budget.check()
55
+ batch = rows[start : start + self._execution.batch_size]
56
+ outcomes.extend(
57
+ await asyncio.gather(*(self._one(template, row, semaphore) for row in batch))
58
+ )
59
+ if progress is not None:
60
+ progress(len(outcomes), len(rows))
61
+ return outcomes
62
+
63
+ async def _one(
64
+ self, template: PromptTemplate, row: Row, semaphore: asyncio.Semaphore
65
+ ) -> ExampleOutcome:
66
+ expected = (
67
+ canonical_expected(row.expected, self._output.type)
68
+ if row.expected is not None
69
+ else None
70
+ )
71
+ request = LLMRequest.for_model(self._spec, render_messages(template, row.inputs))
72
+ async with semaphore:
73
+ try:
74
+ response = await self._client.complete(request)
75
+ except LLMError as exc:
76
+ if self._budget is not None:
77
+ self._budget.record_failed_call()
78
+ return ExampleOutcome(
79
+ row.id,
80
+ expected,
81
+ "",
82
+ None,
83
+ error=ErrorCode.PROVIDER_ERROR,
84
+ detail=redact(str(exc)),
85
+ )
86
+ if self._budget is not None:
87
+ self._budget.record(response)
88
+ return self._outcome(row, expected, response)
89
+
90
+ def _outcome(self, row: Row, expected: str | None, response: LLMResponse) -> ExampleOutcome:
91
+ parsed = parse_output(response.text, response.finish_reason, self._output, self._schema)
92
+ usage = response.usage
93
+ return ExampleOutcome(
94
+ row_id=row.id,
95
+ expected=expected,
96
+ raw_output=response.text,
97
+ value=parsed.value,
98
+ parsed=parsed.parsed,
99
+ error=parsed.error,
100
+ finish_reason=response.finish_reason,
101
+ latency_s=response.latency_s,
102
+ prompt_tokens=usage.prompt_tokens,
103
+ completion_tokens=usage.completion_tokens,
104
+ total_tokens=usage.total_tokens,
105
+ cost_usd=usage.cost_usd,
106
+ cached=response.cached,
107
+ )
strop/formatting.py ADDED
@@ -0,0 +1,14 @@
1
+ """Small text helpers shared by services and the terminal views."""
2
+
3
+ MAX_TEXT = 70
4
+
5
+
6
+ def fmt(value: float | None, digits: int = 4) -> str:
7
+ """A number with fixed decimals, or ``n/a`` when it is unavailable."""
8
+ return "n/a" if value is None else f"{value:.{digits}f}"
9
+
10
+
11
+ def shorten(text: str, limit: int = MAX_TEXT) -> str:
12
+ """Collapse whitespace and cut ``text`` to ``limit`` characters."""
13
+ flat = " ".join(text.split())
14
+ return flat if len(flat) <= limit else flat[: limit - 3] + "..."
strop/llm/__init__.py ADDED
@@ -0,0 +1,7 @@
1
+ """Model access behind a provider-independent interface."""
2
+
3
+ from strop.llm.client import LLMClient
4
+ from strop.llm.mock import MockLLMClient
5
+ from strop.llm.types import LLMRequest, LLMResponse, Message, Usage
6
+
7
+ __all__ = ["LLMClient", "LLMRequest", "LLMResponse", "Message", "MockLLMClient", "Usage"]
strop/llm/caching.py ADDED
@@ -0,0 +1,28 @@
1
+ """Response caching around any client."""
2
+
3
+ from strop.llm.client import LLMClient
4
+ from strop.llm.types import LLMRequest, LLMResponse
5
+ from strop.storage.cache import ResponseCache, cache_key
6
+
7
+
8
+ class CachingClient:
9
+ """Serves repeated deterministic requests from the cache.
10
+
11
+ Only requests at temperature 0 are cached: sampling is meant to vary between calls,
12
+ so a cached answer would silently remove that variety.
13
+ """
14
+
15
+ def __init__(self, inner: LLMClient, cache: ResponseCache) -> None:
16
+ self._inner = inner
17
+ self._cache = cache
18
+
19
+ async def complete(self, request: LLMRequest) -> LLMResponse:
20
+ if request.temperature > 0:
21
+ return await self._inner.complete(request)
22
+ key = cache_key(request)
23
+ hit = self._cache.get(key)
24
+ if hit is not None:
25
+ return hit
26
+ response = await self._inner.complete(request)
27
+ self._cache.put(key, response)
28
+ return response
strop/llm/client.py ADDED
@@ -0,0 +1,15 @@
1
+ """The interface every model provider implementation satisfies."""
2
+
3
+ from typing import Protocol, runtime_checkable
4
+
5
+ from strop.llm.types import LLMRequest, LLMResponse
6
+
7
+
8
+ @runtime_checkable
9
+ class LLMClient(Protocol):
10
+ """Sends one chat request and returns a normalized response.
11
+
12
+ Implementations raise ``strop.errors.LLMError`` when a call fails for good.
13
+ """
14
+
15
+ async def complete(self, request: LLMRequest) -> LLMResponse: ...
strop/llm/factory.py ADDED
@@ -0,0 +1,14 @@
1
+ """Assembling the production client stack."""
2
+
3
+ from strop.config.limits import ExecutionConfig
4
+ from strop.llm.caching import CachingClient
5
+ from strop.llm.client import LLMClient
6
+ from strop.llm.litellm_client import LiteLLMClient
7
+ from strop.llm.retry import RetryingClient
8
+ from strop.storage.cache import ResponseCache
9
+
10
+
11
+ def build_client(execution: ExecutionConfig, cache: ResponseCache | None = None) -> LLMClient:
12
+ """LiteLLM with retries, and a response cache when ``cache`` is given."""
13
+ client: LLMClient = RetryingClient(LiteLLMClient(), attempts=execution.retry_attempts)
14
+ return CachingClient(client, cache) if cache is not None else client
@@ -0,0 +1,90 @@
1
+ """The LiteLLM-backed client. LiteLLM is imported lazily to keep CLI start-up fast."""
2
+
3
+ import logging
4
+ import os
5
+ import time
6
+ from types import ModuleType
7
+ from typing import Any
8
+
9
+ from strop.errors import LLMError, TransientLLMError
10
+ from strop.llm.types import LLMRequest, LLMResponse, Usage
11
+ from strop.log import redact
12
+
13
+ _TRANSIENT_NAMES = (
14
+ "RateLimitError",
15
+ "Timeout",
16
+ "APIConnectionError",
17
+ "ServiceUnavailableError",
18
+ "InternalServerError",
19
+ )
20
+
21
+
22
+ def _load_litellm() -> ModuleType:
23
+ # Use the price table bundled with LiteLLM instead of downloading one when it is imported.
24
+ os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True")
25
+ import litellm # noqa: PLC0415 - deferred: importing litellm takes over a second
26
+
27
+ litellm.suppress_debug_info = True
28
+ litellm.telemetry = False
29
+ logging.getLogger("LiteLLM").setLevel(logging.ERROR)
30
+ return litellm
31
+
32
+
33
+ class LiteLLMClient:
34
+ """Calls any provider and model LiteLLM supports and normalizes the response."""
35
+
36
+ async def complete(self, request: LLMRequest) -> LLMResponse:
37
+ litellm = _load_litellm()
38
+ started = time.perf_counter()
39
+ try:
40
+ raw = await litellm.acompletion(
41
+ model=request.model,
42
+ messages=[{"role": m.role, "content": m.content} for m in request.messages],
43
+ temperature=request.temperature,
44
+ max_tokens=request.max_tokens,
45
+ timeout=request.timeout_s,
46
+ drop_params=True,
47
+ **request.params,
48
+ )
49
+ except Exception as exc:
50
+ raise _translate(litellm, exc) from exc
51
+ return _normalize(litellm, raw, request, time.perf_counter() - started)
52
+
53
+
54
+ def _translate(litellm: ModuleType, exc: Exception) -> LLMError:
55
+ transient = tuple(
56
+ cls for name in _TRANSIENT_NAMES if isinstance(cls := getattr(litellm, name, None), type)
57
+ )
58
+ message = redact(f"{type(exc).__name__}: {exc}")
59
+ return TransientLLMError(message) if isinstance(exc, transient) else LLMError(message)
60
+
61
+
62
+ def _normalize(litellm: ModuleType, raw: Any, request: LLMRequest, latency_s: float) -> LLMResponse:
63
+ try:
64
+ choice = raw.choices[0]
65
+ text = choice.message.content or ""
66
+ finish_reason = choice.finish_reason
67
+ except (AttributeError, IndexError, TypeError) as exc:
68
+ raise LLMError("the provider returned a response without a message") from exc
69
+ return LLMResponse(
70
+ text=text,
71
+ model=getattr(raw, "model", None) or request.model,
72
+ finish_reason=finish_reason,
73
+ usage=_usage(litellm, raw),
74
+ latency_s=latency_s,
75
+ request_id=getattr(raw, "id", None),
76
+ )
77
+
78
+
79
+ def _usage(litellm: ModuleType, raw: Any) -> Usage:
80
+ reported = getattr(raw, "usage", None)
81
+ try:
82
+ cost = float(litellm.completion_cost(completion_response=raw))
83
+ except Exception:
84
+ cost = None
85
+ return Usage(
86
+ prompt_tokens=getattr(reported, "prompt_tokens", None),
87
+ completion_tokens=getattr(reported, "completion_tokens", None),
88
+ total_tokens=getattr(reported, "total_tokens", None),
89
+ cost_usd=cost,
90
+ )