strop-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- strop/__init__.py +5 -0
- strop/__main__.py +5 -0
- strop/analysis/__init__.py +17 -0
- strop/analysis/classify.py +52 -0
- strop/analysis/evidence.py +52 -0
- strop/analysis/facts.py +74 -0
- strop/analysis/patterns.py +90 -0
- strop/cli/__init__.py +5 -0
- strop/cli/analyze.py +93 -0
- strop/cli/app.py +82 -0
- strop/cli/data.py +111 -0
- strop/cli/evaluate.py +145 -0
- strop/cli/export.py +74 -0
- strop/cli/final.py +106 -0
- strop/cli/init.py +131 -0
- strop/cli/optimize.py +156 -0
- strop/cli/options.py +20 -0
- strop/cli/progress.py +18 -0
- strop/cli/prompts.py +110 -0
- strop/cli/status.py +68 -0
- strop/cli/versions.py +123 -0
- strop/config/__init__.py +16 -0
- strop/config/base.py +9 -0
- strop/config/constraints.py +33 -0
- strop/config/dataset.py +63 -0
- strop/config/enums.py +52 -0
- strop/config/io.py +52 -0
- strop/config/limits.py +19 -0
- strop/config/llm.py +42 -0
- strop/config/metrics.py +86 -0
- strop/config/optimization.py +23 -0
- strop/config/output.py +31 -0
- strop/config/overrides.py +22 -0
- strop/config/root.py +36 -0
- strop/config/rules.py +67 -0
- strop/config/split.py +23 -0
- strop/data/__init__.py +58 -0
- strop/data/loaders.py +85 -0
- strop/data/models.py +44 -0
- strop/data/rows.py +99 -0
- strop/data/splitting.py +103 -0
- strop/env.py +44 -0
- strop/errors.py +59 -0
- strop/evaluation/__init__.py +1 -0
- strop/evaluation/budget.py +67 -0
- strop/evaluation/budgeted.py +23 -0
- strop/evaluation/examples.py +49 -0
- strop/evaluation/models.py +105 -0
- strop/evaluation/parsing.py +96 -0
- strop/evaluation/runner.py +107 -0
- strop/formatting.py +14 -0
- strop/llm/__init__.py +7 -0
- strop/llm/caching.py +28 -0
- strop/llm/client.py +15 -0
- strop/llm/factory.py +14 -0
- strop/llm/litellm_client.py +90 -0
- strop/llm/mock.py +43 -0
- strop/llm/retry.py +44 -0
- strop/llm/structured.py +52 -0
- strop/llm/types.py +59 -0
- strop/log.py +49 -0
- strop/metrics/__init__.py +1 -0
- strop/metrics/basic.py +30 -0
- strop/metrics/classification.py +164 -0
- strop/metrics/judge.py +130 -0
- strop/metrics/python_metric.py +101 -0
- strop/metrics/python_runner.py +65 -0
- strop/metrics/registry.py +115 -0
- strop/metrics/rules.py +55 -0
- strop/metrics/structured.py +109 -0
- strop/metrics/suite.py +109 -0
- strop/metrics/text.py +55 -0
- strop/metrics/types.py +106 -0
- strop/optimization/__init__.py +25 -0
- strop/optimization/acceptance.py +170 -0
- strop/optimization/generator.py +92 -0
- strop/optimization/meta_prompt.py +101 -0
- strop/optimization/types.py +69 -0
- strop/prompts/__init__.py +18 -0
- strop/prompts/diff.py +139 -0
- strop/prompts/files.py +92 -0
- strop/prompts/models.py +51 -0
- strop/prompts/render.py +16 -0
- strop/prompts/template.py +45 -0
- strop/prompts/validation.py +31 -0
- strop/prompts/versions.py +34 -0
- strop/py.typed +0 -0
- strop/reporting/__init__.py +6 -0
- strop/reporting/console.py +20 -0
- strop/reporting/diff_view.py +102 -0
- strop/reporting/evaluation_view.py +141 -0
- strop/reporting/final_view.py +109 -0
- strop/reporting/prompt_export.py +58 -0
- strop/reporting/report_markdown.py +157 -0
- strop/reporting/run_view.py +118 -0
- strop/services/__init__.py +1 -0
- strop/services/analysis.py +124 -0
- strop/services/clean.py +38 -0
- strop/services/compare.py +60 -0
- strop/services/context.py +70 -0
- strop/services/data.py +65 -0
- strop/services/evaluate.py +105 -0
- strop/services/evaluator.py +169 -0
- strop/services/export.py +90 -0
- strop/services/final_test.py +69 -0
- strop/services/metrics.py +89 -0
- strop/services/optimize.py +187 -0
- strop/services/partitions.py +46 -0
- strop/services/project.py +55 -0
- strop/services/prompts.py +76 -0
- strop/services/report.py +194 -0
- strop/services/review.py +43 -0
- strop/services/scaffold.py +79 -0
- strop/services/turn.py +234 -0
- strop/storage/__init__.py +6 -0
- strop/storage/cache.py +45 -0
- strop/storage/database.py +45 -0
- strop/storage/datasets.py +129 -0
- strop/storage/evaluations.py +160 -0
- strop/storage/migrations/001_initial.sql +5 -0
- strop/storage/migrations/002_datasets.sql +27 -0
- strop/storage/migrations/003_prompt_versions.sql +22 -0
- strop/storage/migrations/004_llm_cache.sql +6 -0
- strop/storage/migrations/005_evaluations.sql +37 -0
- strop/storage/migrations/006_runs.sql +52 -0
- strop/storage/migrations/007_example_feedback.sql +2 -0
- strop/storage/migrations/__init__.py +5 -0
- strop/storage/migrator.py +91 -0
- strop/storage/paths.py +30 -0
- strop/storage/prompts.py +96 -0
- strop/storage/run_models.py +75 -0
- strop/storage/runs.py +263 -0
- strop_cli-0.1.0.dist-info/METADATA +242 -0
- strop_cli-0.1.0.dist-info/RECORD +137 -0
- strop_cli-0.1.0.dist-info/WHEEL +4 -0
- strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
- strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Optimization loop settings."""
|
|
2
|
+
|
|
3
|
+
from pydantic import Field
|
|
4
|
+
|
|
5
|
+
from strop.config.base import StrictModel
|
|
6
|
+
from strop.config.enums import ApprovalMode
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class OptimizationConfig(StrictModel):
|
|
10
|
+
max_turns: int = Field(default=5, ge=1)
|
|
11
|
+
candidates_per_turn: int = Field(default=3, ge=1)
|
|
12
|
+
target: float | None = None
|
|
13
|
+
patience: int = Field(default=2, ge=1)
|
|
14
|
+
approval: ApprovalMode = ApprovalMode.AUTO
|
|
15
|
+
min_improvement: float = Field(default=0.01, ge=0)
|
|
16
|
+
train_sample: int | None = Field(default=None, gt=0)
|
|
17
|
+
instructions: str = ""
|
|
18
|
+
evidence_examples: int = Field(default=12, ge=1)
|
|
19
|
+
evidence_chars: int = Field(default=400, ge=40)
|
|
20
|
+
max_class_regression: float | None = Field(default=0.10, ge=0)
|
|
21
|
+
max_validity_regression: float = Field(default=0.02, ge=0)
|
|
22
|
+
min_validation_samples: int = Field(default=10, ge=1)
|
|
23
|
+
require_paired_wins: bool = True
|
strop/config/output.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Output contract: what the prompted model must return."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Literal, Self
|
|
5
|
+
|
|
6
|
+
from pydantic import model_validator
|
|
7
|
+
|
|
8
|
+
from strop.config.base import StrictModel
|
|
9
|
+
from strop.config.enums import OutputType
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class OutputConfig(StrictModel):
|
|
13
|
+
type: OutputType
|
|
14
|
+
labels: tuple[str, ...] = ()
|
|
15
|
+
schema_file: Path | None = None
|
|
16
|
+
match: Literal["strict", "contains"] = "strict"
|
|
17
|
+
|
|
18
|
+
@model_validator(mode="after")
|
|
19
|
+
def _check_contract(self) -> Self:
|
|
20
|
+
if self.type is OutputType.LABEL:
|
|
21
|
+
if len(self.labels) < 2:
|
|
22
|
+
raise ValueError("output.labels needs at least two labels for type 'label'")
|
|
23
|
+
if len(set(self.labels)) != len(self.labels):
|
|
24
|
+
raise ValueError("output.labels must not contain duplicates")
|
|
25
|
+
elif self.labels:
|
|
26
|
+
raise ValueError("output.labels is only valid for type 'label'")
|
|
27
|
+
if self.schema_file is not None and self.type is not OutputType.JSON:
|
|
28
|
+
raise ValueError("output.schema_file is only valid for type 'json'")
|
|
29
|
+
if self.match != "strict" and self.type is not OutputType.LABEL:
|
|
30
|
+
raise ValueError("output.match is only valid for type 'label'")
|
|
31
|
+
return self
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Applying command-line overrides on top of a configuration section."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Mapping
|
|
4
|
+
from typing import Any, TypeVar
|
|
5
|
+
|
|
6
|
+
from pydantic import ValidationError
|
|
7
|
+
|
|
8
|
+
from strop.config.base import StrictModel
|
|
9
|
+
from strop.config.io import format_errors
|
|
10
|
+
from strop.errors import ConfigError
|
|
11
|
+
|
|
12
|
+
T = TypeVar("T", bound=StrictModel)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def apply_overrides(section: T, overrides: Mapping[str, Any]) -> T:
|
|
16
|
+
"""A validated copy of ``section`` with every non-None override applied."""
|
|
17
|
+
values = section.model_dump()
|
|
18
|
+
values.update({key: value for key, value in overrides.items() if value is not None})
|
|
19
|
+
try:
|
|
20
|
+
return type(section).model_validate(values)
|
|
21
|
+
except ValidationError as exc:
|
|
22
|
+
raise ConfigError(f"invalid options:\n{format_errors(exc)}") from exc
|
strop/config/root.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""The complete project configuration stored in strop.yaml."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from pydantic import Field
|
|
6
|
+
|
|
7
|
+
from strop.config.base import StrictModel
|
|
8
|
+
from strop.config.dataset import DatasetConfig
|
|
9
|
+
from strop.config.limits import BudgetsConfig, ExecutionConfig
|
|
10
|
+
from strop.config.llm import ModelsConfig
|
|
11
|
+
from strop.config.metrics import MetricsConfig
|
|
12
|
+
from strop.config.optimization import OptimizationConfig
|
|
13
|
+
from strop.config.output import OutputConfig
|
|
14
|
+
from strop.config.split import SplitConfig
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ProjectConfig(StrictModel):
|
|
18
|
+
name: str = Field(min_length=1)
|
|
19
|
+
task_description: str = Field(min_length=1)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class PromptConfig(StrictModel):
|
|
23
|
+
file: Path = Path("prompt.yaml")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class StropConfig(StrictModel):
|
|
27
|
+
project: ProjectConfig
|
|
28
|
+
prompt: PromptConfig = Field(default_factory=PromptConfig)
|
|
29
|
+
dataset: DatasetConfig
|
|
30
|
+
output: OutputConfig
|
|
31
|
+
models: ModelsConfig
|
|
32
|
+
metrics: MetricsConfig
|
|
33
|
+
split: SplitConfig = Field(default_factory=SplitConfig)
|
|
34
|
+
optimization: OptimizationConfig = Field(default_factory=OptimizationConfig)
|
|
35
|
+
budgets: BudgetsConfig = Field(default_factory=BudgetsConfig)
|
|
36
|
+
execution: ExecutionConfig = Field(default_factory=ExecutionConfig)
|
strop/config/rules.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Parsing of ``--rule`` flags and rubric files used by ``strop init``."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Iterable
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from pydantic import ValidationError
|
|
8
|
+
from ruamel.yaml import YAML
|
|
9
|
+
from ruamel.yaml.error import YAMLError
|
|
10
|
+
|
|
11
|
+
from strop.config.enums import RuleType
|
|
12
|
+
from strop.config.io import format_errors
|
|
13
|
+
from strop.config.metrics import JudgeSpec, RuleSpec
|
|
14
|
+
from strop.errors import ConfigError
|
|
15
|
+
|
|
16
|
+
_COUNT = {RuleType.MAX_CHARS, RuleType.MIN_CHARS, RuleType.MAX_WORDS, RuleType.MIN_WORDS}
|
|
17
|
+
_LISTS = {RuleType.CONTAINS_ALL, RuleType.CONTAINS_NONE}
|
|
18
|
+
FORM = "use the form name=type:value, for example short=max_words:120"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def parse_rules(expressions: Iterable[str]) -> dict[str, RuleSpec]:
|
|
22
|
+
"""Turn ``name=type:value`` expressions into rule specs."""
|
|
23
|
+
rules: dict[str, RuleSpec] = {}
|
|
24
|
+
for text in expressions:
|
|
25
|
+
name, equals, rest = text.partition("=")
|
|
26
|
+
kind, colon, raw = rest.partition(":")
|
|
27
|
+
if not (equals and colon and name.strip() and raw):
|
|
28
|
+
raise ConfigError(f"invalid rule '{text}'; {FORM}")
|
|
29
|
+
if name.strip() in rules:
|
|
30
|
+
raise ConfigError(f"rule '{name.strip()}' is defined more than once")
|
|
31
|
+
rules[name.strip()] = _spec(text, kind.strip(), raw)
|
|
32
|
+
return rules
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _spec(text: str, kind: str, raw: str) -> RuleSpec:
|
|
36
|
+
try:
|
|
37
|
+
rule_type = RuleType(kind)
|
|
38
|
+
except ValueError:
|
|
39
|
+
known = ", ".join(t.value for t in RuleType)
|
|
40
|
+
raise ConfigError(f"unknown rule type '{kind}' in '{text}'; types: {known}") from None
|
|
41
|
+
value: Any = raw
|
|
42
|
+
if rule_type in _COUNT:
|
|
43
|
+
if not raw.strip().isdigit():
|
|
44
|
+
raise ConfigError(f"rule '{text}' needs a whole number after the colon")
|
|
45
|
+
value = int(raw)
|
|
46
|
+
elif rule_type in _LISTS:
|
|
47
|
+
value = [item.strip() for item in raw.split(",") if item.strip()]
|
|
48
|
+
try:
|
|
49
|
+
return RuleSpec(type=rule_type, value=value)
|
|
50
|
+
except ValidationError as exc:
|
|
51
|
+
raise ConfigError(f"invalid rule '{text}':\n{format_errors(exc)}") from exc
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def load_rubrics(path: Path) -> dict[str, JudgeSpec]:
|
|
55
|
+
"""Read a YAML file mapping rubric names to ``criteria``, ``scale_min``, ``scale_max``."""
|
|
56
|
+
try:
|
|
57
|
+
data = YAML(typ="safe").load(path.read_text(encoding="utf-8"))
|
|
58
|
+
except OSError as exc:
|
|
59
|
+
raise ConfigError(f"cannot read the rubric file {path}: {exc.strerror}") from exc
|
|
60
|
+
except YAMLError as exc:
|
|
61
|
+
raise ConfigError(f"{path.name} is not valid YAML: {exc}") from exc
|
|
62
|
+
if not isinstance(data, dict) or not data:
|
|
63
|
+
raise ConfigError(f"{path.name} must map each rubric name to its settings")
|
|
64
|
+
try:
|
|
65
|
+
return {str(name): JudgeSpec.model_validate(spec) for name, spec in data.items()}
|
|
66
|
+
except ValidationError as exc:
|
|
67
|
+
raise ConfigError(f"invalid rubric in {path.name}:\n{format_errors(exc)}") from exc
|
strop/config/split.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Train, validation, and test partition settings."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
from typing import Self
|
|
5
|
+
|
|
6
|
+
from pydantic import Field, model_validator
|
|
7
|
+
|
|
8
|
+
from strop.config.base import StrictModel
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class SplitConfig(StrictModel):
|
|
12
|
+
train: float = Field(default=0.6, gt=0, lt=1)
|
|
13
|
+
validation: float = Field(default=0.2, gt=0, lt=1)
|
|
14
|
+
test: float = Field(default=0.2, ge=0, lt=1)
|
|
15
|
+
seed: int = 42
|
|
16
|
+
stratify: bool = True
|
|
17
|
+
|
|
18
|
+
@model_validator(mode="after")
|
|
19
|
+
def _check_ratios(self) -> Self:
|
|
20
|
+
total = self.train + self.validation + self.test
|
|
21
|
+
if not math.isclose(total, 1.0, abs_tol=1e-6):
|
|
22
|
+
raise ValueError(f"split ratios must sum to 1.0, got {total:.3f}")
|
|
23
|
+
return self
|
strop/data/__init__.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Dataset loading, validation, and splitting."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from strop.config import StropConfig
|
|
6
|
+
from strop.data.loaders import file_hash, read_records
|
|
7
|
+
from strop.data.models import Dataset, Partition, Row, RowIssue
|
|
8
|
+
from strop.data.rows import build_rows
|
|
9
|
+
from strop.data.splitting import SplitSummary, split_rows, summarize
|
|
10
|
+
from strop.errors import DataError
|
|
11
|
+
|
|
12
|
+
MAX_LISTED_ISSUES = 20
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def load_dataset(config: StropConfig, root: Path) -> Dataset:
|
|
16
|
+
"""Read and validate the configured dataset. Invalid rows are reported, not dropped silently."""
|
|
17
|
+
path = root / config.dataset.path
|
|
18
|
+
columns, records = read_records(path, config.dataset.resolved_format)
|
|
19
|
+
wanted = [
|
|
20
|
+
*config.dataset.columns,
|
|
21
|
+
*([config.dataset.id_column] if config.dataset.id_column else []),
|
|
22
|
+
]
|
|
23
|
+
missing = [name for name in wanted if name not in columns]
|
|
24
|
+
if missing:
|
|
25
|
+
raise DataError(
|
|
26
|
+
f"{path.name} has no column named {', '.join(repr(m) for m in missing)}; "
|
|
27
|
+
f"available columns: {', '.join(columns)}"
|
|
28
|
+
)
|
|
29
|
+
rows, issues = build_rows(records, config.dataset, config.output)
|
|
30
|
+
return Dataset(tuple(rows), tuple(issues), len(records), file_hash(path), columns)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def require_valid(dataset: Dataset, *, skip_invalid: bool) -> Dataset:
|
|
34
|
+
"""Return ``dataset`` if it has no issues or ``skip_invalid`` is set; otherwise raise."""
|
|
35
|
+
if dataset.issues and not skip_invalid:
|
|
36
|
+
shown = "\n".join(f" {issue}" for issue in dataset.issues[:MAX_LISTED_ISSUES])
|
|
37
|
+
extra = len(dataset.issues) - MAX_LISTED_ISSUES
|
|
38
|
+
more = f"\n ... and {extra} more" if extra > 0 else ""
|
|
39
|
+
raise DataError(
|
|
40
|
+
f"{len(dataset.issues)} invalid row(s) in the dataset:\n{shown}{more}\n"
|
|
41
|
+
"fix the file, or pass --skip-invalid to exclude these rows"
|
|
42
|
+
)
|
|
43
|
+
if not dataset.rows:
|
|
44
|
+
raise DataError("the dataset has no valid rows")
|
|
45
|
+
return dataset
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
__all__ = [
|
|
49
|
+
"Dataset",
|
|
50
|
+
"Partition",
|
|
51
|
+
"Row",
|
|
52
|
+
"RowIssue",
|
|
53
|
+
"SplitSummary",
|
|
54
|
+
"load_dataset",
|
|
55
|
+
"require_valid",
|
|
56
|
+
"split_rows",
|
|
57
|
+
"summarize",
|
|
58
|
+
]
|
strop/data/loaders.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""Reading CSV and JSONL files into raw records."""
|
|
2
|
+
|
|
3
|
+
import csv
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from strop.config import DatasetFormat
|
|
11
|
+
from strop.errors import DataError
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True, slots=True)
|
|
15
|
+
class RawRecord:
|
|
16
|
+
"""One file record before validation. ``error`` is set when it could not be parsed."""
|
|
17
|
+
|
|
18
|
+
number: int
|
|
19
|
+
values: dict[str, Any]
|
|
20
|
+
error: str | None = None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def file_hash(path: Path) -> str:
|
|
24
|
+
"""SHA-256 of the raw file bytes."""
|
|
25
|
+
digest = hashlib.sha256()
|
|
26
|
+
with path.open("rb") as handle:
|
|
27
|
+
for chunk in iter(lambda: handle.read(1 << 20), b""):
|
|
28
|
+
digest.update(chunk)
|
|
29
|
+
return digest.hexdigest()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def read_records(path: Path, fmt: DatasetFormat) -> tuple[tuple[str, ...], list[RawRecord]]:
|
|
33
|
+
"""Return the column names and every record of ``path``."""
|
|
34
|
+
try:
|
|
35
|
+
if fmt is DatasetFormat.CSV:
|
|
36
|
+
return _read_csv(path)
|
|
37
|
+
return _read_jsonl(path)
|
|
38
|
+
except FileNotFoundError:
|
|
39
|
+
raise DataError(f"dataset file not found: {path}") from None
|
|
40
|
+
except UnicodeDecodeError as exc:
|
|
41
|
+
raise DataError(f"{path.name} is not valid UTF-8 text: {exc.reason}") from exc
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _read_csv(path: Path) -> tuple[tuple[str, ...], list[RawRecord]]:
|
|
45
|
+
with path.open(encoding="utf-8-sig", newline="") as handle:
|
|
46
|
+
reader = csv.reader(handle)
|
|
47
|
+
header = next(reader, None)
|
|
48
|
+
if not header or not any(name.strip() for name in header):
|
|
49
|
+
raise DataError(f"{path.name} has no header row")
|
|
50
|
+
columns = tuple(name.strip() for name in header)
|
|
51
|
+
if len(set(columns)) != len(columns):
|
|
52
|
+
raise DataError(f"{path.name} has duplicate column names in its header")
|
|
53
|
+
records = [_csv_record(number, columns, cells) for number, cells in enumerate(reader, 1)]
|
|
54
|
+
return columns, records
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _csv_record(number: int, columns: tuple[str, ...], cells: list[str]) -> RawRecord:
|
|
58
|
+
if len(cells) != len(columns):
|
|
59
|
+
return RawRecord(number, {}, f"has {len(cells)} fields, expected {len(columns)}")
|
|
60
|
+
return RawRecord(number, dict(zip(columns, cells, strict=True)))
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _read_jsonl(path: Path) -> tuple[tuple[str, ...], list[RawRecord]]:
|
|
64
|
+
columns: dict[str, None] = {}
|
|
65
|
+
records = []
|
|
66
|
+
with path.open(encoding="utf-8-sig") as handle:
|
|
67
|
+
for number, line in enumerate(handle, 1):
|
|
68
|
+
if not line.strip():
|
|
69
|
+
continue
|
|
70
|
+
record = _jsonl_record(number, line)
|
|
71
|
+
columns.update(dict.fromkeys(record.values))
|
|
72
|
+
records.append(record)
|
|
73
|
+
if not records:
|
|
74
|
+
raise DataError(f"{path.name} contains no records")
|
|
75
|
+
return tuple(columns), records
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _jsonl_record(number: int, line: str) -> RawRecord:
|
|
79
|
+
try:
|
|
80
|
+
value = json.loads(line)
|
|
81
|
+
except json.JSONDecodeError as exc:
|
|
82
|
+
return RawRecord(number, {}, f"is not valid JSON ({exc.msg})")
|
|
83
|
+
if not isinstance(value, dict):
|
|
84
|
+
return RawRecord(number, {}, "is not a JSON object")
|
|
85
|
+
return RawRecord(number, value)
|
strop/data/models.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Dataset domain types."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from enum import StrEnum
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Partition(StrEnum):
|
|
8
|
+
TRAIN = "train"
|
|
9
|
+
VALIDATION = "validation"
|
|
10
|
+
TEST = "test"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(frozen=True, slots=True)
|
|
14
|
+
class Row:
|
|
15
|
+
"""One valid dataset row, reduced to the columns Strop uses."""
|
|
16
|
+
|
|
17
|
+
id: str
|
|
18
|
+
number: int
|
|
19
|
+
inputs: dict[str, str]
|
|
20
|
+
expected: str | None
|
|
21
|
+
metadata: dict[str, str]
|
|
22
|
+
hash: str
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True, slots=True)
|
|
26
|
+
class RowIssue:
|
|
27
|
+
"""A problem with one record; ``number`` is its CSV record or JSONL line number."""
|
|
28
|
+
|
|
29
|
+
number: int
|
|
30
|
+
message: str
|
|
31
|
+
|
|
32
|
+
def __str__(self) -> str:
|
|
33
|
+
return f"row {self.number}: {self.message}"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class Dataset:
|
|
38
|
+
"""A loaded dataset: the valid rows plus every issue found, never silently dropped."""
|
|
39
|
+
|
|
40
|
+
rows: tuple[Row, ...]
|
|
41
|
+
issues: tuple[RowIssue, ...]
|
|
42
|
+
total_records: int
|
|
43
|
+
content_hash: str
|
|
44
|
+
columns: tuple[str, ...]
|
strop/data/rows.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Turning raw records into validated rows."""
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
from collections.abc import Iterable
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from strop.config.dataset import DatasetConfig
|
|
9
|
+
from strop.config.enums import OutputType
|
|
10
|
+
from strop.config.output import OutputConfig
|
|
11
|
+
from strop.data.loaders import RawRecord
|
|
12
|
+
from strop.data.models import Row, RowIssue
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def to_text(value: Any) -> str:
|
|
16
|
+
"""Normalize a cell to text: strings as-is, JSON-style for everything else."""
|
|
17
|
+
if isinstance(value, str):
|
|
18
|
+
return value
|
|
19
|
+
if value is None:
|
|
20
|
+
return ""
|
|
21
|
+
if isinstance(value, bool | int | float | dict | list):
|
|
22
|
+
return json.dumps(value, ensure_ascii=False, sort_keys=True)
|
|
23
|
+
return str(value)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def row_hash(inputs: dict[str, str], expected: str | None) -> str:
|
|
27
|
+
"""Stable hash of the content a model call or metric depends on."""
|
|
28
|
+
payload = json.dumps(
|
|
29
|
+
{"inputs": inputs, "expected": expected}, sort_keys=True, ensure_ascii=False
|
|
30
|
+
)
|
|
31
|
+
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def build_rows(
|
|
35
|
+
records: Iterable[RawRecord], dataset: DatasetConfig, output: OutputConfig
|
|
36
|
+
) -> tuple[list[Row], list[RowIssue]]:
|
|
37
|
+
"""Validate ``records`` and return the valid rows and the issues found."""
|
|
38
|
+
rows: list[Row] = []
|
|
39
|
+
issues: list[RowIssue] = []
|
|
40
|
+
seen: set[str] = set()
|
|
41
|
+
for record in records:
|
|
42
|
+
if record.error:
|
|
43
|
+
issues.append(RowIssue(record.number, record.error))
|
|
44
|
+
continue
|
|
45
|
+
problems = _check_record(record, dataset, output)
|
|
46
|
+
row_id = _row_id(record, dataset)
|
|
47
|
+
if row_id in seen:
|
|
48
|
+
problems.append(f"duplicate id '{row_id}'")
|
|
49
|
+
if problems:
|
|
50
|
+
issues.extend(RowIssue(record.number, text) for text in problems)
|
|
51
|
+
continue
|
|
52
|
+
seen.add(row_id)
|
|
53
|
+
rows.append(_make_row(record, row_id, dataset))
|
|
54
|
+
return rows, issues
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _row_id(record: RawRecord, dataset: DatasetConfig) -> str:
|
|
58
|
+
if dataset.id_column is None:
|
|
59
|
+
return f"row_{record.number:04d}"
|
|
60
|
+
return to_text(record.values.get(dataset.id_column)).strip()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _check_record(record: RawRecord, dataset: DatasetConfig, output: OutputConfig) -> list[str]:
|
|
64
|
+
problems = []
|
|
65
|
+
if dataset.id_column and not _row_id(record, dataset):
|
|
66
|
+
problems.append(f"id column '{dataset.id_column}' is empty")
|
|
67
|
+
for column in dataset.input_columns:
|
|
68
|
+
if not to_text(record.values.get(column)).strip():
|
|
69
|
+
problems.append(f"input column '{column}' is empty")
|
|
70
|
+
expected_column = dataset.expected_column
|
|
71
|
+
if expected_column:
|
|
72
|
+
expected = to_text(record.values.get(expected_column)).strip()
|
|
73
|
+
if not expected:
|
|
74
|
+
problems.append(f"expected column '{expected_column}' is empty")
|
|
75
|
+
elif output.type is OutputType.LABEL and expected not in output.labels:
|
|
76
|
+
problems.append(f"expected value '{expected}' is not one of the output labels")
|
|
77
|
+
elif output.type is OutputType.JSON and not _is_json(expected):
|
|
78
|
+
problems.append(f"expected column '{expected_column}' does not contain valid JSON")
|
|
79
|
+
return problems
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _is_json(text: str) -> bool:
|
|
83
|
+
try:
|
|
84
|
+
json.loads(text)
|
|
85
|
+
except ValueError:
|
|
86
|
+
return False
|
|
87
|
+
return True
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _make_row(record: RawRecord, row_id: str, dataset: DatasetConfig) -> Row:
|
|
91
|
+
inputs = {name: to_text(record.values.get(name)) for name in dataset.input_columns}
|
|
92
|
+
expected_column = dataset.expected_column
|
|
93
|
+
expected = to_text(record.values.get(expected_column)).strip() if expected_column else None
|
|
94
|
+
metadata = {
|
|
95
|
+
name: to_text(record.values.get(name))
|
|
96
|
+
for name, role in dataset.columns.items()
|
|
97
|
+
if role.value == "metadata"
|
|
98
|
+
}
|
|
99
|
+
return Row(row_id, record.number, inputs, expected, metadata, row_hash(inputs, expected))
|
strop/data/splitting.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Deterministic, optionally stratified train/validation/test splits."""
|
|
2
|
+
|
|
3
|
+
import random
|
|
4
|
+
from collections import defaultdict
|
|
5
|
+
from collections.abc import Sequence
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
|
|
8
|
+
from strop.config.split import SplitConfig
|
|
9
|
+
from strop.data.models import Partition, Row
|
|
10
|
+
from strop.errors import DataError
|
|
11
|
+
|
|
12
|
+
_ORDER = (Partition.TRAIN, Partition.VALIDATION, Partition.TEST)
|
|
13
|
+
MIN_RELIABLE_VALIDATION = 30
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class SplitSummary:
|
|
18
|
+
counts: dict[Partition, int]
|
|
19
|
+
warnings: list[str] = field(default_factory=list)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def stratify_groups(rows: Sequence[Row], config: SplitConfig) -> dict[str, list[Row]]:
|
|
23
|
+
"""Group rows by expected value, or into one group when grouping would not help.
|
|
24
|
+
|
|
25
|
+
Stratifying only makes sense for class-like values that repeat. If more than half of the
|
|
26
|
+
groups would hold a single row (free text, JSON objects, numbers), the split is random.
|
|
27
|
+
"""
|
|
28
|
+
ordered = sorted(rows, key=lambda r: r.id)
|
|
29
|
+
groups: dict[str, list[Row]] = defaultdict(list)
|
|
30
|
+
if config.stratify:
|
|
31
|
+
for row in ordered:
|
|
32
|
+
groups[row.expected if row.expected is not None else ""].append(row)
|
|
33
|
+
if not groups or len(groups) * 2 > len(rows):
|
|
34
|
+
return {"": ordered}
|
|
35
|
+
return dict(groups)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def split_rows(rows: Sequence[Row], config: SplitConfig) -> dict[str, Partition]:
|
|
39
|
+
"""Assign every row to a partition. The same rows and config always give the same result."""
|
|
40
|
+
ratios = (config.train, config.validation, config.test)
|
|
41
|
+
groups = stratify_groups(rows, config)
|
|
42
|
+
rng = random.Random(config.seed)
|
|
43
|
+
assignments: dict[str, Partition] = {}
|
|
44
|
+
for key in sorted(groups):
|
|
45
|
+
members = groups[key]
|
|
46
|
+
rng.shuffle(members)
|
|
47
|
+
start = 0
|
|
48
|
+
for partition, size in zip(_ORDER, _allocate(len(members), ratios), strict=True):
|
|
49
|
+
for row in members[start : start + size]:
|
|
50
|
+
assignments[row.id] = partition
|
|
51
|
+
start += size
|
|
52
|
+
|
|
53
|
+
present = set(assignments.values())
|
|
54
|
+
missing = [p.value for p, r in zip(_ORDER, ratios, strict=True) if r > 0 and p not in present]
|
|
55
|
+
if missing:
|
|
56
|
+
raise DataError(
|
|
57
|
+
f"{len(rows)} rows are too few for this split: the {', '.join(missing)} "
|
|
58
|
+
"partition would be empty; add data or change the split ratios"
|
|
59
|
+
)
|
|
60
|
+
return assignments
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def summarize(
|
|
64
|
+
rows: Sequence[Row], assignments: dict[str, Partition], config: SplitConfig
|
|
65
|
+
) -> SplitSummary:
|
|
66
|
+
"""Partition sizes and warnings about splits that give unreliable comparisons."""
|
|
67
|
+
counts = dict.fromkeys(_ORDER, 0)
|
|
68
|
+
for partition in assignments.values():
|
|
69
|
+
counts[partition] += 1
|
|
70
|
+
warnings = []
|
|
71
|
+
if counts[Partition.VALIDATION] < MIN_RELIABLE_VALIDATION:
|
|
72
|
+
warnings.append(
|
|
73
|
+
f"the validation partition has {counts[Partition.VALIDATION]} rows "
|
|
74
|
+
f"(fewer than {MIN_RELIABLE_VALIDATION}); score differences may be noise"
|
|
75
|
+
)
|
|
76
|
+
if counts[Partition.TEST] == 0:
|
|
77
|
+
warnings.append(
|
|
78
|
+
"there is no test partition; the final result cannot be checked on unseen data"
|
|
79
|
+
)
|
|
80
|
+
if len(stratify_groups(rows, config)) > 1:
|
|
81
|
+
labels = {r.expected for r in rows if r.expected is not None}
|
|
82
|
+
for partition in (Partition.TRAIN, Partition.VALIDATION):
|
|
83
|
+
seen = {r.expected for r in rows if assignments[r.id] is partition}
|
|
84
|
+
absent = sorted(labels - seen)
|
|
85
|
+
if absent:
|
|
86
|
+
warnings.append(f"no '{partition.value}' rows for: {', '.join(absent)}")
|
|
87
|
+
return SplitSummary(counts, warnings)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _allocate(size: int, ratios: tuple[float, float, float]) -> list[int]:
|
|
91
|
+
"""Split ``size`` into three counts by largest remainder, keeping rare classes represented."""
|
|
92
|
+
ideal = [size * ratio for ratio in ratios]
|
|
93
|
+
counts = [int(value) for value in ideal]
|
|
94
|
+
by_remainder = sorted(range(3), key=lambda i: (-(ideal[i] - counts[i]), i))
|
|
95
|
+
for index in by_remainder[: size - sum(counts)]:
|
|
96
|
+
counts[index] += 1
|
|
97
|
+
for index in (1, 2):
|
|
98
|
+
if ratios[index] > 0 and counts[index] == 0 and size > index:
|
|
99
|
+
donor = max(range(3), key=lambda i: counts[i])
|
|
100
|
+
if counts[donor] > 1:
|
|
101
|
+
counts[donor] -= 1
|
|
102
|
+
counts[index] += 1
|
|
103
|
+
return counts
|
strop/env.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Loading API keys and other settings from a project's .env file."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from strop.log import LOGGER_NAME
|
|
9
|
+
|
|
10
|
+
LOG = logging.getLogger(LOGGER_NAME)
|
|
11
|
+
_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def load_env_file(path: Path) -> list[str]:
|
|
15
|
+
"""Set variables from ``path`` that are not already in the environment.
|
|
16
|
+
|
|
17
|
+
The format is ``NAME=value`` per line, with optional ``export`` prefix, ``#`` comments, and
|
|
18
|
+
single or double quotes around the value. Variables already set are never overridden, and
|
|
19
|
+
values are never logged. Returns the names that were set.
|
|
20
|
+
"""
|
|
21
|
+
try:
|
|
22
|
+
text = path.read_text(encoding="utf-8")
|
|
23
|
+
except OSError:
|
|
24
|
+
return []
|
|
25
|
+
loaded = []
|
|
26
|
+
for raw in text.splitlines():
|
|
27
|
+
line = raw.strip().removeprefix("export ").strip()
|
|
28
|
+
if not line or line.startswith("#") or "=" not in line:
|
|
29
|
+
continue
|
|
30
|
+
name, _, value = line.partition("=")
|
|
31
|
+
name = name.strip()
|
|
32
|
+
if not _NAME.match(name) or name in os.environ:
|
|
33
|
+
continue
|
|
34
|
+
os.environ[name] = _unquote(value.strip())
|
|
35
|
+
loaded.append(name)
|
|
36
|
+
if loaded:
|
|
37
|
+
LOG.debug("loaded %d variable(s) from %s", len(loaded), path.name)
|
|
38
|
+
return loaded
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _unquote(value: str) -> str:
|
|
42
|
+
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
|
|
43
|
+
return value[1:-1]
|
|
44
|
+
return value.split(" #", 1)[0].rstrip()
|