strop-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. strop/__init__.py +5 -0
  2. strop/__main__.py +5 -0
  3. strop/analysis/__init__.py +17 -0
  4. strop/analysis/classify.py +52 -0
  5. strop/analysis/evidence.py +52 -0
  6. strop/analysis/facts.py +74 -0
  7. strop/analysis/patterns.py +90 -0
  8. strop/cli/__init__.py +5 -0
  9. strop/cli/analyze.py +93 -0
  10. strop/cli/app.py +82 -0
  11. strop/cli/data.py +111 -0
  12. strop/cli/evaluate.py +145 -0
  13. strop/cli/export.py +74 -0
  14. strop/cli/final.py +106 -0
  15. strop/cli/init.py +131 -0
  16. strop/cli/optimize.py +156 -0
  17. strop/cli/options.py +20 -0
  18. strop/cli/progress.py +18 -0
  19. strop/cli/prompts.py +110 -0
  20. strop/cli/status.py +68 -0
  21. strop/cli/versions.py +123 -0
  22. strop/config/__init__.py +16 -0
  23. strop/config/base.py +9 -0
  24. strop/config/constraints.py +33 -0
  25. strop/config/dataset.py +63 -0
  26. strop/config/enums.py +52 -0
  27. strop/config/io.py +52 -0
  28. strop/config/limits.py +19 -0
  29. strop/config/llm.py +42 -0
  30. strop/config/metrics.py +86 -0
  31. strop/config/optimization.py +23 -0
  32. strop/config/output.py +31 -0
  33. strop/config/overrides.py +22 -0
  34. strop/config/root.py +36 -0
  35. strop/config/rules.py +67 -0
  36. strop/config/split.py +23 -0
  37. strop/data/__init__.py +58 -0
  38. strop/data/loaders.py +85 -0
  39. strop/data/models.py +44 -0
  40. strop/data/rows.py +99 -0
  41. strop/data/splitting.py +103 -0
  42. strop/env.py +44 -0
  43. strop/errors.py +59 -0
  44. strop/evaluation/__init__.py +1 -0
  45. strop/evaluation/budget.py +67 -0
  46. strop/evaluation/budgeted.py +23 -0
  47. strop/evaluation/examples.py +49 -0
  48. strop/evaluation/models.py +105 -0
  49. strop/evaluation/parsing.py +96 -0
  50. strop/evaluation/runner.py +107 -0
  51. strop/formatting.py +14 -0
  52. strop/llm/__init__.py +7 -0
  53. strop/llm/caching.py +28 -0
  54. strop/llm/client.py +15 -0
  55. strop/llm/factory.py +14 -0
  56. strop/llm/litellm_client.py +90 -0
  57. strop/llm/mock.py +43 -0
  58. strop/llm/retry.py +44 -0
  59. strop/llm/structured.py +52 -0
  60. strop/llm/types.py +59 -0
  61. strop/log.py +49 -0
  62. strop/metrics/__init__.py +1 -0
  63. strop/metrics/basic.py +30 -0
  64. strop/metrics/classification.py +164 -0
  65. strop/metrics/judge.py +130 -0
  66. strop/metrics/python_metric.py +101 -0
  67. strop/metrics/python_runner.py +65 -0
  68. strop/metrics/registry.py +115 -0
  69. strop/metrics/rules.py +55 -0
  70. strop/metrics/structured.py +109 -0
  71. strop/metrics/suite.py +109 -0
  72. strop/metrics/text.py +55 -0
  73. strop/metrics/types.py +106 -0
  74. strop/optimization/__init__.py +25 -0
  75. strop/optimization/acceptance.py +170 -0
  76. strop/optimization/generator.py +92 -0
  77. strop/optimization/meta_prompt.py +101 -0
  78. strop/optimization/types.py +69 -0
  79. strop/prompts/__init__.py +18 -0
  80. strop/prompts/diff.py +139 -0
  81. strop/prompts/files.py +92 -0
  82. strop/prompts/models.py +51 -0
  83. strop/prompts/render.py +16 -0
  84. strop/prompts/template.py +45 -0
  85. strop/prompts/validation.py +31 -0
  86. strop/prompts/versions.py +34 -0
  87. strop/py.typed +0 -0
  88. strop/reporting/__init__.py +6 -0
  89. strop/reporting/console.py +20 -0
  90. strop/reporting/diff_view.py +102 -0
  91. strop/reporting/evaluation_view.py +141 -0
  92. strop/reporting/final_view.py +109 -0
  93. strop/reporting/prompt_export.py +58 -0
  94. strop/reporting/report_markdown.py +157 -0
  95. strop/reporting/run_view.py +118 -0
  96. strop/services/__init__.py +1 -0
  97. strop/services/analysis.py +124 -0
  98. strop/services/clean.py +38 -0
  99. strop/services/compare.py +60 -0
  100. strop/services/context.py +70 -0
  101. strop/services/data.py +65 -0
  102. strop/services/evaluate.py +105 -0
  103. strop/services/evaluator.py +169 -0
  104. strop/services/export.py +90 -0
  105. strop/services/final_test.py +69 -0
  106. strop/services/metrics.py +89 -0
  107. strop/services/optimize.py +187 -0
  108. strop/services/partitions.py +46 -0
  109. strop/services/project.py +55 -0
  110. strop/services/prompts.py +76 -0
  111. strop/services/report.py +194 -0
  112. strop/services/review.py +43 -0
  113. strop/services/scaffold.py +79 -0
  114. strop/services/turn.py +234 -0
  115. strop/storage/__init__.py +6 -0
  116. strop/storage/cache.py +45 -0
  117. strop/storage/database.py +45 -0
  118. strop/storage/datasets.py +129 -0
  119. strop/storage/evaluations.py +160 -0
  120. strop/storage/migrations/001_initial.sql +5 -0
  121. strop/storage/migrations/002_datasets.sql +27 -0
  122. strop/storage/migrations/003_prompt_versions.sql +22 -0
  123. strop/storage/migrations/004_llm_cache.sql +6 -0
  124. strop/storage/migrations/005_evaluations.sql +37 -0
  125. strop/storage/migrations/006_runs.sql +52 -0
  126. strop/storage/migrations/007_example_feedback.sql +2 -0
  127. strop/storage/migrations/__init__.py +5 -0
  128. strop/storage/migrator.py +91 -0
  129. strop/storage/paths.py +30 -0
  130. strop/storage/prompts.py +96 -0
  131. strop/storage/run_models.py +75 -0
  132. strop/storage/runs.py +263 -0
  133. strop_cli-0.1.0.dist-info/METADATA +242 -0
  134. strop_cli-0.1.0.dist-info/RECORD +137 -0
  135. strop_cli-0.1.0.dist-info/WHEEL +4 -0
  136. strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
  137. strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,23 @@
1
+ """Optimization loop settings."""
2
+
3
+ from pydantic import Field
4
+
5
+ from strop.config.base import StrictModel
6
+ from strop.config.enums import ApprovalMode
7
+
8
+
9
+ class OptimizationConfig(StrictModel):
10
+ max_turns: int = Field(default=5, ge=1)
11
+ candidates_per_turn: int = Field(default=3, ge=1)
12
+ target: float | None = None
13
+ patience: int = Field(default=2, ge=1)
14
+ approval: ApprovalMode = ApprovalMode.AUTO
15
+ min_improvement: float = Field(default=0.01, ge=0)
16
+ train_sample: int | None = Field(default=None, gt=0)
17
+ instructions: str = ""
18
+ evidence_examples: int = Field(default=12, ge=1)
19
+ evidence_chars: int = Field(default=400, ge=40)
20
+ max_class_regression: float | None = Field(default=0.10, ge=0)
21
+ max_validity_regression: float = Field(default=0.02, ge=0)
22
+ min_validation_samples: int = Field(default=10, ge=1)
23
+ require_paired_wins: bool = True
strop/config/output.py ADDED
@@ -0,0 +1,31 @@
1
+ """Output contract: what the prompted model must return."""
2
+
3
+ from pathlib import Path
4
+ from typing import Literal, Self
5
+
6
+ from pydantic import model_validator
7
+
8
+ from strop.config.base import StrictModel
9
+ from strop.config.enums import OutputType
10
+
11
+
12
+ class OutputConfig(StrictModel):
13
+ type: OutputType
14
+ labels: tuple[str, ...] = ()
15
+ schema_file: Path | None = None
16
+ match: Literal["strict", "contains"] = "strict"
17
+
18
+ @model_validator(mode="after")
19
+ def _check_contract(self) -> Self:
20
+ if self.type is OutputType.LABEL:
21
+ if len(self.labels) < 2:
22
+ raise ValueError("output.labels needs at least two labels for type 'label'")
23
+ if len(set(self.labels)) != len(self.labels):
24
+ raise ValueError("output.labels must not contain duplicates")
25
+ elif self.labels:
26
+ raise ValueError("output.labels is only valid for type 'label'")
27
+ if self.schema_file is not None and self.type is not OutputType.JSON:
28
+ raise ValueError("output.schema_file is only valid for type 'json'")
29
+ if self.match != "strict" and self.type is not OutputType.LABEL:
30
+ raise ValueError("output.match is only valid for type 'label'")
31
+ return self
@@ -0,0 +1,22 @@
1
+ """Applying command-line overrides on top of a configuration section."""
2
+
3
+ from collections.abc import Mapping
4
+ from typing import Any, TypeVar
5
+
6
+ from pydantic import ValidationError
7
+
8
+ from strop.config.base import StrictModel
9
+ from strop.config.io import format_errors
10
+ from strop.errors import ConfigError
11
+
12
+ T = TypeVar("T", bound=StrictModel)
13
+
14
+
15
+ def apply_overrides(section: T, overrides: Mapping[str, Any]) -> T:
16
+ """A validated copy of ``section`` with every non-None override applied."""
17
+ values = section.model_dump()
18
+ values.update({key: value for key, value in overrides.items() if value is not None})
19
+ try:
20
+ return type(section).model_validate(values)
21
+ except ValidationError as exc:
22
+ raise ConfigError(f"invalid options:\n{format_errors(exc)}") from exc
strop/config/root.py ADDED
@@ -0,0 +1,36 @@
1
+ """The complete project configuration stored in strop.yaml."""
2
+
3
+ from pathlib import Path
4
+
5
+ from pydantic import Field
6
+
7
+ from strop.config.base import StrictModel
8
+ from strop.config.dataset import DatasetConfig
9
+ from strop.config.limits import BudgetsConfig, ExecutionConfig
10
+ from strop.config.llm import ModelsConfig
11
+ from strop.config.metrics import MetricsConfig
12
+ from strop.config.optimization import OptimizationConfig
13
+ from strop.config.output import OutputConfig
14
+ from strop.config.split import SplitConfig
15
+
16
+
17
+ class ProjectConfig(StrictModel):
18
+ name: str = Field(min_length=1)
19
+ task_description: str = Field(min_length=1)
20
+
21
+
22
+ class PromptConfig(StrictModel):
23
+ file: Path = Path("prompt.yaml")
24
+
25
+
26
+ class StropConfig(StrictModel):
27
+ project: ProjectConfig
28
+ prompt: PromptConfig = Field(default_factory=PromptConfig)
29
+ dataset: DatasetConfig
30
+ output: OutputConfig
31
+ models: ModelsConfig
32
+ metrics: MetricsConfig
33
+ split: SplitConfig = Field(default_factory=SplitConfig)
34
+ optimization: OptimizationConfig = Field(default_factory=OptimizationConfig)
35
+ budgets: BudgetsConfig = Field(default_factory=BudgetsConfig)
36
+ execution: ExecutionConfig = Field(default_factory=ExecutionConfig)
strop/config/rules.py ADDED
@@ -0,0 +1,67 @@
1
+ """Parsing of ``--rule`` flags and rubric files used by ``strop init``."""
2
+
3
+ from collections.abc import Iterable
4
+ from pathlib import Path
5
+ from typing import Any
6
+
7
+ from pydantic import ValidationError
8
+ from ruamel.yaml import YAML
9
+ from ruamel.yaml.error import YAMLError
10
+
11
+ from strop.config.enums import RuleType
12
+ from strop.config.io import format_errors
13
+ from strop.config.metrics import JudgeSpec, RuleSpec
14
+ from strop.errors import ConfigError
15
+
16
+ _COUNT = {RuleType.MAX_CHARS, RuleType.MIN_CHARS, RuleType.MAX_WORDS, RuleType.MIN_WORDS}
17
+ _LISTS = {RuleType.CONTAINS_ALL, RuleType.CONTAINS_NONE}
18
+ FORM = "use the form name=type:value, for example short=max_words:120"
19
+
20
+
21
+ def parse_rules(expressions: Iterable[str]) -> dict[str, RuleSpec]:
22
+ """Turn ``name=type:value`` expressions into rule specs."""
23
+ rules: dict[str, RuleSpec] = {}
24
+ for text in expressions:
25
+ name, equals, rest = text.partition("=")
26
+ kind, colon, raw = rest.partition(":")
27
+ if not (equals and colon and name.strip() and raw):
28
+ raise ConfigError(f"invalid rule '{text}'; {FORM}")
29
+ if name.strip() in rules:
30
+ raise ConfigError(f"rule '{name.strip()}' is defined more than once")
31
+ rules[name.strip()] = _spec(text, kind.strip(), raw)
32
+ return rules
33
+
34
+
35
+ def _spec(text: str, kind: str, raw: str) -> RuleSpec:
36
+ try:
37
+ rule_type = RuleType(kind)
38
+ except ValueError:
39
+ known = ", ".join(t.value for t in RuleType)
40
+ raise ConfigError(f"unknown rule type '{kind}' in '{text}'; types: {known}") from None
41
+ value: Any = raw
42
+ if rule_type in _COUNT:
43
+ if not raw.strip().isdigit():
44
+ raise ConfigError(f"rule '{text}' needs a whole number after the colon")
45
+ value = int(raw)
46
+ elif rule_type in _LISTS:
47
+ value = [item.strip() for item in raw.split(",") if item.strip()]
48
+ try:
49
+ return RuleSpec(type=rule_type, value=value)
50
+ except ValidationError as exc:
51
+ raise ConfigError(f"invalid rule '{text}':\n{format_errors(exc)}") from exc
52
+
53
+
54
+ def load_rubrics(path: Path) -> dict[str, JudgeSpec]:
55
+ """Read a YAML file mapping rubric names to ``criteria``, ``scale_min``, ``scale_max``."""
56
+ try:
57
+ data = YAML(typ="safe").load(path.read_text(encoding="utf-8"))
58
+ except OSError as exc:
59
+ raise ConfigError(f"cannot read the rubric file {path}: {exc.strerror}") from exc
60
+ except YAMLError as exc:
61
+ raise ConfigError(f"{path.name} is not valid YAML: {exc}") from exc
62
+ if not isinstance(data, dict) or not data:
63
+ raise ConfigError(f"{path.name} must map each rubric name to its settings")
64
+ try:
65
+ return {str(name): JudgeSpec.model_validate(spec) for name, spec in data.items()}
66
+ except ValidationError as exc:
67
+ raise ConfigError(f"invalid rubric in {path.name}:\n{format_errors(exc)}") from exc
strop/config/split.py ADDED
@@ -0,0 +1,23 @@
1
+ """Train, validation, and test partition settings."""
2
+
3
+ import math
4
+ from typing import Self
5
+
6
+ from pydantic import Field, model_validator
7
+
8
+ from strop.config.base import StrictModel
9
+
10
+
11
+ class SplitConfig(StrictModel):
12
+ train: float = Field(default=0.6, gt=0, lt=1)
13
+ validation: float = Field(default=0.2, gt=0, lt=1)
14
+ test: float = Field(default=0.2, ge=0, lt=1)
15
+ seed: int = 42
16
+ stratify: bool = True
17
+
18
+ @model_validator(mode="after")
19
+ def _check_ratios(self) -> Self:
20
+ total = self.train + self.validation + self.test
21
+ if not math.isclose(total, 1.0, abs_tol=1e-6):
22
+ raise ValueError(f"split ratios must sum to 1.0, got {total:.3f}")
23
+ return self
strop/data/__init__.py ADDED
@@ -0,0 +1,58 @@
1
+ """Dataset loading, validation, and splitting."""
2
+
3
+ from pathlib import Path
4
+
5
+ from strop.config import StropConfig
6
+ from strop.data.loaders import file_hash, read_records
7
+ from strop.data.models import Dataset, Partition, Row, RowIssue
8
+ from strop.data.rows import build_rows
9
+ from strop.data.splitting import SplitSummary, split_rows, summarize
10
+ from strop.errors import DataError
11
+
12
+ MAX_LISTED_ISSUES = 20
13
+
14
+
15
+ def load_dataset(config: StropConfig, root: Path) -> Dataset:
16
+ """Read and validate the configured dataset. Invalid rows are reported, not dropped silently."""
17
+ path = root / config.dataset.path
18
+ columns, records = read_records(path, config.dataset.resolved_format)
19
+ wanted = [
20
+ *config.dataset.columns,
21
+ *([config.dataset.id_column] if config.dataset.id_column else []),
22
+ ]
23
+ missing = [name for name in wanted if name not in columns]
24
+ if missing:
25
+ raise DataError(
26
+ f"{path.name} has no column named {', '.join(repr(m) for m in missing)}; "
27
+ f"available columns: {', '.join(columns)}"
28
+ )
29
+ rows, issues = build_rows(records, config.dataset, config.output)
30
+ return Dataset(tuple(rows), tuple(issues), len(records), file_hash(path), columns)
31
+
32
+
33
+ def require_valid(dataset: Dataset, *, skip_invalid: bool) -> Dataset:
34
+ """Return ``dataset`` if it has no issues or ``skip_invalid`` is set; otherwise raise."""
35
+ if dataset.issues and not skip_invalid:
36
+ shown = "\n".join(f" {issue}" for issue in dataset.issues[:MAX_LISTED_ISSUES])
37
+ extra = len(dataset.issues) - MAX_LISTED_ISSUES
38
+ more = f"\n ... and {extra} more" if extra > 0 else ""
39
+ raise DataError(
40
+ f"{len(dataset.issues)} invalid row(s) in the dataset:\n{shown}{more}\n"
41
+ "fix the file, or pass --skip-invalid to exclude these rows"
42
+ )
43
+ if not dataset.rows:
44
+ raise DataError("the dataset has no valid rows")
45
+ return dataset
46
+
47
+
48
+ __all__ = [
49
+ "Dataset",
50
+ "Partition",
51
+ "Row",
52
+ "RowIssue",
53
+ "SplitSummary",
54
+ "load_dataset",
55
+ "require_valid",
56
+ "split_rows",
57
+ "summarize",
58
+ ]
strop/data/loaders.py ADDED
@@ -0,0 +1,85 @@
1
+ """Reading CSV and JSONL files into raw records."""
2
+
3
+ import csv
4
+ import hashlib
5
+ import json
6
+ from dataclasses import dataclass
7
+ from pathlib import Path
8
+ from typing import Any
9
+
10
+ from strop.config import DatasetFormat
11
+ from strop.errors import DataError
12
+
13
+
14
+ @dataclass(frozen=True, slots=True)
15
+ class RawRecord:
16
+ """One file record before validation. ``error`` is set when it could not be parsed."""
17
+
18
+ number: int
19
+ values: dict[str, Any]
20
+ error: str | None = None
21
+
22
+
23
+ def file_hash(path: Path) -> str:
24
+ """SHA-256 of the raw file bytes."""
25
+ digest = hashlib.sha256()
26
+ with path.open("rb") as handle:
27
+ for chunk in iter(lambda: handle.read(1 << 20), b""):
28
+ digest.update(chunk)
29
+ return digest.hexdigest()
30
+
31
+
32
+ def read_records(path: Path, fmt: DatasetFormat) -> tuple[tuple[str, ...], list[RawRecord]]:
33
+ """Return the column names and every record of ``path``."""
34
+ try:
35
+ if fmt is DatasetFormat.CSV:
36
+ return _read_csv(path)
37
+ return _read_jsonl(path)
38
+ except FileNotFoundError:
39
+ raise DataError(f"dataset file not found: {path}") from None
40
+ except UnicodeDecodeError as exc:
41
+ raise DataError(f"{path.name} is not valid UTF-8 text: {exc.reason}") from exc
42
+
43
+
44
+ def _read_csv(path: Path) -> tuple[tuple[str, ...], list[RawRecord]]:
45
+ with path.open(encoding="utf-8-sig", newline="") as handle:
46
+ reader = csv.reader(handle)
47
+ header = next(reader, None)
48
+ if not header or not any(name.strip() for name in header):
49
+ raise DataError(f"{path.name} has no header row")
50
+ columns = tuple(name.strip() for name in header)
51
+ if len(set(columns)) != len(columns):
52
+ raise DataError(f"{path.name} has duplicate column names in its header")
53
+ records = [_csv_record(number, columns, cells) for number, cells in enumerate(reader, 1)]
54
+ return columns, records
55
+
56
+
57
+ def _csv_record(number: int, columns: tuple[str, ...], cells: list[str]) -> RawRecord:
58
+ if len(cells) != len(columns):
59
+ return RawRecord(number, {}, f"has {len(cells)} fields, expected {len(columns)}")
60
+ return RawRecord(number, dict(zip(columns, cells, strict=True)))
61
+
62
+
63
+ def _read_jsonl(path: Path) -> tuple[tuple[str, ...], list[RawRecord]]:
64
+ columns: dict[str, None] = {}
65
+ records = []
66
+ with path.open(encoding="utf-8-sig") as handle:
67
+ for number, line in enumerate(handle, 1):
68
+ if not line.strip():
69
+ continue
70
+ record = _jsonl_record(number, line)
71
+ columns.update(dict.fromkeys(record.values))
72
+ records.append(record)
73
+ if not records:
74
+ raise DataError(f"{path.name} contains no records")
75
+ return tuple(columns), records
76
+
77
+
78
+ def _jsonl_record(number: int, line: str) -> RawRecord:
79
+ try:
80
+ value = json.loads(line)
81
+ except json.JSONDecodeError as exc:
82
+ return RawRecord(number, {}, f"is not valid JSON ({exc.msg})")
83
+ if not isinstance(value, dict):
84
+ return RawRecord(number, {}, "is not a JSON object")
85
+ return RawRecord(number, value)
strop/data/models.py ADDED
@@ -0,0 +1,44 @@
1
+ """Dataset domain types."""
2
+
3
+ from dataclasses import dataclass
4
+ from enum import StrEnum
5
+
6
+
7
+ class Partition(StrEnum):
8
+ TRAIN = "train"
9
+ VALIDATION = "validation"
10
+ TEST = "test"
11
+
12
+
13
+ @dataclass(frozen=True, slots=True)
14
+ class Row:
15
+ """One valid dataset row, reduced to the columns Strop uses."""
16
+
17
+ id: str
18
+ number: int
19
+ inputs: dict[str, str]
20
+ expected: str | None
21
+ metadata: dict[str, str]
22
+ hash: str
23
+
24
+
25
+ @dataclass(frozen=True, slots=True)
26
+ class RowIssue:
27
+ """A problem with one record; ``number`` is its CSV record or JSONL line number."""
28
+
29
+ number: int
30
+ message: str
31
+
32
+ def __str__(self) -> str:
33
+ return f"row {self.number}: {self.message}"
34
+
35
+
36
+ @dataclass(frozen=True)
37
+ class Dataset:
38
+ """A loaded dataset: the valid rows plus every issue found, never silently dropped."""
39
+
40
+ rows: tuple[Row, ...]
41
+ issues: tuple[RowIssue, ...]
42
+ total_records: int
43
+ content_hash: str
44
+ columns: tuple[str, ...]
strop/data/rows.py ADDED
@@ -0,0 +1,99 @@
1
+ """Turning raw records into validated rows."""
2
+
3
+ import hashlib
4
+ import json
5
+ from collections.abc import Iterable
6
+ from typing import Any
7
+
8
+ from strop.config.dataset import DatasetConfig
9
+ from strop.config.enums import OutputType
10
+ from strop.config.output import OutputConfig
11
+ from strop.data.loaders import RawRecord
12
+ from strop.data.models import Row, RowIssue
13
+
14
+
15
+ def to_text(value: Any) -> str:
16
+ """Normalize a cell to text: strings as-is, JSON-style for everything else."""
17
+ if isinstance(value, str):
18
+ return value
19
+ if value is None:
20
+ return ""
21
+ if isinstance(value, bool | int | float | dict | list):
22
+ return json.dumps(value, ensure_ascii=False, sort_keys=True)
23
+ return str(value)
24
+
25
+
26
+ def row_hash(inputs: dict[str, str], expected: str | None) -> str:
27
+ """Stable hash of the content a model call or metric depends on."""
28
+ payload = json.dumps(
29
+ {"inputs": inputs, "expected": expected}, sort_keys=True, ensure_ascii=False
30
+ )
31
+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()
32
+
33
+
34
+ def build_rows(
35
+ records: Iterable[RawRecord], dataset: DatasetConfig, output: OutputConfig
36
+ ) -> tuple[list[Row], list[RowIssue]]:
37
+ """Validate ``records`` and return the valid rows and the issues found."""
38
+ rows: list[Row] = []
39
+ issues: list[RowIssue] = []
40
+ seen: set[str] = set()
41
+ for record in records:
42
+ if record.error:
43
+ issues.append(RowIssue(record.number, record.error))
44
+ continue
45
+ problems = _check_record(record, dataset, output)
46
+ row_id = _row_id(record, dataset)
47
+ if row_id in seen:
48
+ problems.append(f"duplicate id '{row_id}'")
49
+ if problems:
50
+ issues.extend(RowIssue(record.number, text) for text in problems)
51
+ continue
52
+ seen.add(row_id)
53
+ rows.append(_make_row(record, row_id, dataset))
54
+ return rows, issues
55
+
56
+
57
+ def _row_id(record: RawRecord, dataset: DatasetConfig) -> str:
58
+ if dataset.id_column is None:
59
+ return f"row_{record.number:04d}"
60
+ return to_text(record.values.get(dataset.id_column)).strip()
61
+
62
+
63
+ def _check_record(record: RawRecord, dataset: DatasetConfig, output: OutputConfig) -> list[str]:
64
+ problems = []
65
+ if dataset.id_column and not _row_id(record, dataset):
66
+ problems.append(f"id column '{dataset.id_column}' is empty")
67
+ for column in dataset.input_columns:
68
+ if not to_text(record.values.get(column)).strip():
69
+ problems.append(f"input column '{column}' is empty")
70
+ expected_column = dataset.expected_column
71
+ if expected_column:
72
+ expected = to_text(record.values.get(expected_column)).strip()
73
+ if not expected:
74
+ problems.append(f"expected column '{expected_column}' is empty")
75
+ elif output.type is OutputType.LABEL and expected not in output.labels:
76
+ problems.append(f"expected value '{expected}' is not one of the output labels")
77
+ elif output.type is OutputType.JSON and not _is_json(expected):
78
+ problems.append(f"expected column '{expected_column}' does not contain valid JSON")
79
+ return problems
80
+
81
+
82
+ def _is_json(text: str) -> bool:
83
+ try:
84
+ json.loads(text)
85
+ except ValueError:
86
+ return False
87
+ return True
88
+
89
+
90
+ def _make_row(record: RawRecord, row_id: str, dataset: DatasetConfig) -> Row:
91
+ inputs = {name: to_text(record.values.get(name)) for name in dataset.input_columns}
92
+ expected_column = dataset.expected_column
93
+ expected = to_text(record.values.get(expected_column)).strip() if expected_column else None
94
+ metadata = {
95
+ name: to_text(record.values.get(name))
96
+ for name, role in dataset.columns.items()
97
+ if role.value == "metadata"
98
+ }
99
+ return Row(row_id, record.number, inputs, expected, metadata, row_hash(inputs, expected))
@@ -0,0 +1,103 @@
1
+ """Deterministic, optionally stratified train/validation/test splits."""
2
+
3
+ import random
4
+ from collections import defaultdict
5
+ from collections.abc import Sequence
6
+ from dataclasses import dataclass, field
7
+
8
+ from strop.config.split import SplitConfig
9
+ from strop.data.models import Partition, Row
10
+ from strop.errors import DataError
11
+
12
+ _ORDER = (Partition.TRAIN, Partition.VALIDATION, Partition.TEST)
13
+ MIN_RELIABLE_VALIDATION = 30
14
+
15
+
16
+ @dataclass(frozen=True)
17
+ class SplitSummary:
18
+ counts: dict[Partition, int]
19
+ warnings: list[str] = field(default_factory=list)
20
+
21
+
22
+ def stratify_groups(rows: Sequence[Row], config: SplitConfig) -> dict[str, list[Row]]:
23
+ """Group rows by expected value, or into one group when grouping would not help.
24
+
25
+ Stratifying only makes sense for class-like values that repeat. If more than half of the
26
+ groups would hold a single row (free text, JSON objects, numbers), the split is random.
27
+ """
28
+ ordered = sorted(rows, key=lambda r: r.id)
29
+ groups: dict[str, list[Row]] = defaultdict(list)
30
+ if config.stratify:
31
+ for row in ordered:
32
+ groups[row.expected if row.expected is not None else ""].append(row)
33
+ if not groups or len(groups) * 2 > len(rows):
34
+ return {"": ordered}
35
+ return dict(groups)
36
+
37
+
38
+ def split_rows(rows: Sequence[Row], config: SplitConfig) -> dict[str, Partition]:
39
+ """Assign every row to a partition. The same rows and config always give the same result."""
40
+ ratios = (config.train, config.validation, config.test)
41
+ groups = stratify_groups(rows, config)
42
+ rng = random.Random(config.seed)
43
+ assignments: dict[str, Partition] = {}
44
+ for key in sorted(groups):
45
+ members = groups[key]
46
+ rng.shuffle(members)
47
+ start = 0
48
+ for partition, size in zip(_ORDER, _allocate(len(members), ratios), strict=True):
49
+ for row in members[start : start + size]:
50
+ assignments[row.id] = partition
51
+ start += size
52
+
53
+ present = set(assignments.values())
54
+ missing = [p.value for p, r in zip(_ORDER, ratios, strict=True) if r > 0 and p not in present]
55
+ if missing:
56
+ raise DataError(
57
+ f"{len(rows)} rows are too few for this split: the {', '.join(missing)} "
58
+ "partition would be empty; add data or change the split ratios"
59
+ )
60
+ return assignments
61
+
62
+
63
+ def summarize(
64
+ rows: Sequence[Row], assignments: dict[str, Partition], config: SplitConfig
65
+ ) -> SplitSummary:
66
+ """Partition sizes and warnings about splits that give unreliable comparisons."""
67
+ counts = dict.fromkeys(_ORDER, 0)
68
+ for partition in assignments.values():
69
+ counts[partition] += 1
70
+ warnings = []
71
+ if counts[Partition.VALIDATION] < MIN_RELIABLE_VALIDATION:
72
+ warnings.append(
73
+ f"the validation partition has {counts[Partition.VALIDATION]} rows "
74
+ f"(fewer than {MIN_RELIABLE_VALIDATION}); score differences may be noise"
75
+ )
76
+ if counts[Partition.TEST] == 0:
77
+ warnings.append(
78
+ "there is no test partition; the final result cannot be checked on unseen data"
79
+ )
80
+ if len(stratify_groups(rows, config)) > 1:
81
+ labels = {r.expected for r in rows if r.expected is not None}
82
+ for partition in (Partition.TRAIN, Partition.VALIDATION):
83
+ seen = {r.expected for r in rows if assignments[r.id] is partition}
84
+ absent = sorted(labels - seen)
85
+ if absent:
86
+ warnings.append(f"no '{partition.value}' rows for: {', '.join(absent)}")
87
+ return SplitSummary(counts, warnings)
88
+
89
+
90
+ def _allocate(size: int, ratios: tuple[float, float, float]) -> list[int]:
91
+ """Split ``size`` into three counts by largest remainder, keeping rare classes represented."""
92
+ ideal = [size * ratio for ratio in ratios]
93
+ counts = [int(value) for value in ideal]
94
+ by_remainder = sorted(range(3), key=lambda i: (-(ideal[i] - counts[i]), i))
95
+ for index in by_remainder[: size - sum(counts)]:
96
+ counts[index] += 1
97
+ for index in (1, 2):
98
+ if ratios[index] > 0 and counts[index] == 0 and size > index:
99
+ donor = max(range(3), key=lambda i: counts[i])
100
+ if counts[donor] > 1:
101
+ counts[donor] -= 1
102
+ counts[index] += 1
103
+ return counts
strop/env.py ADDED
@@ -0,0 +1,44 @@
1
+ """Loading API keys and other settings from a project's .env file."""
2
+
3
+ import logging
4
+ import os
5
+ import re
6
+ from pathlib import Path
7
+
8
+ from strop.log import LOGGER_NAME
9
+
10
+ LOG = logging.getLogger(LOGGER_NAME)
11
+ _NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
12
+
13
+
14
+ def load_env_file(path: Path) -> list[str]:
15
+ """Set variables from ``path`` that are not already in the environment.
16
+
17
+ The format is ``NAME=value`` per line, with optional ``export`` prefix, ``#`` comments, and
18
+ single or double quotes around the value. Variables already set are never overridden, and
19
+ values are never logged. Returns the names that were set.
20
+ """
21
+ try:
22
+ text = path.read_text(encoding="utf-8")
23
+ except OSError:
24
+ return []
25
+ loaded = []
26
+ for raw in text.splitlines():
27
+ line = raw.strip().removeprefix("export ").strip()
28
+ if not line or line.startswith("#") or "=" not in line:
29
+ continue
30
+ name, _, value = line.partition("=")
31
+ name = name.strip()
32
+ if not _NAME.match(name) or name in os.environ:
33
+ continue
34
+ os.environ[name] = _unquote(value.strip())
35
+ loaded.append(name)
36
+ if loaded:
37
+ LOG.debug("loaded %d variable(s) from %s", len(loaded), path.name)
38
+ return loaded
39
+
40
+
41
+ def _unquote(value: str) -> str:
42
+ if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
43
+ return value[1:-1]
44
+ return value.split(" #", 1)[0].rstrip()