strop-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- strop/__init__.py +5 -0
- strop/__main__.py +5 -0
- strop/analysis/__init__.py +17 -0
- strop/analysis/classify.py +52 -0
- strop/analysis/evidence.py +52 -0
- strop/analysis/facts.py +74 -0
- strop/analysis/patterns.py +90 -0
- strop/cli/__init__.py +5 -0
- strop/cli/analyze.py +93 -0
- strop/cli/app.py +82 -0
- strop/cli/data.py +111 -0
- strop/cli/evaluate.py +145 -0
- strop/cli/export.py +74 -0
- strop/cli/final.py +106 -0
- strop/cli/init.py +131 -0
- strop/cli/optimize.py +156 -0
- strop/cli/options.py +20 -0
- strop/cli/progress.py +18 -0
- strop/cli/prompts.py +110 -0
- strop/cli/status.py +68 -0
- strop/cli/versions.py +123 -0
- strop/config/__init__.py +16 -0
- strop/config/base.py +9 -0
- strop/config/constraints.py +33 -0
- strop/config/dataset.py +63 -0
- strop/config/enums.py +52 -0
- strop/config/io.py +52 -0
- strop/config/limits.py +19 -0
- strop/config/llm.py +42 -0
- strop/config/metrics.py +86 -0
- strop/config/optimization.py +23 -0
- strop/config/output.py +31 -0
- strop/config/overrides.py +22 -0
- strop/config/root.py +36 -0
- strop/config/rules.py +67 -0
- strop/config/split.py +23 -0
- strop/data/__init__.py +58 -0
- strop/data/loaders.py +85 -0
- strop/data/models.py +44 -0
- strop/data/rows.py +99 -0
- strop/data/splitting.py +103 -0
- strop/env.py +44 -0
- strop/errors.py +59 -0
- strop/evaluation/__init__.py +1 -0
- strop/evaluation/budget.py +67 -0
- strop/evaluation/budgeted.py +23 -0
- strop/evaluation/examples.py +49 -0
- strop/evaluation/models.py +105 -0
- strop/evaluation/parsing.py +96 -0
- strop/evaluation/runner.py +107 -0
- strop/formatting.py +14 -0
- strop/llm/__init__.py +7 -0
- strop/llm/caching.py +28 -0
- strop/llm/client.py +15 -0
- strop/llm/factory.py +14 -0
- strop/llm/litellm_client.py +90 -0
- strop/llm/mock.py +43 -0
- strop/llm/retry.py +44 -0
- strop/llm/structured.py +52 -0
- strop/llm/types.py +59 -0
- strop/log.py +49 -0
- strop/metrics/__init__.py +1 -0
- strop/metrics/basic.py +30 -0
- strop/metrics/classification.py +164 -0
- strop/metrics/judge.py +130 -0
- strop/metrics/python_metric.py +101 -0
- strop/metrics/python_runner.py +65 -0
- strop/metrics/registry.py +115 -0
- strop/metrics/rules.py +55 -0
- strop/metrics/structured.py +109 -0
- strop/metrics/suite.py +109 -0
- strop/metrics/text.py +55 -0
- strop/metrics/types.py +106 -0
- strop/optimization/__init__.py +25 -0
- strop/optimization/acceptance.py +170 -0
- strop/optimization/generator.py +92 -0
- strop/optimization/meta_prompt.py +101 -0
- strop/optimization/types.py +69 -0
- strop/prompts/__init__.py +18 -0
- strop/prompts/diff.py +139 -0
- strop/prompts/files.py +92 -0
- strop/prompts/models.py +51 -0
- strop/prompts/render.py +16 -0
- strop/prompts/template.py +45 -0
- strop/prompts/validation.py +31 -0
- strop/prompts/versions.py +34 -0
- strop/py.typed +0 -0
- strop/reporting/__init__.py +6 -0
- strop/reporting/console.py +20 -0
- strop/reporting/diff_view.py +102 -0
- strop/reporting/evaluation_view.py +141 -0
- strop/reporting/final_view.py +109 -0
- strop/reporting/prompt_export.py +58 -0
- strop/reporting/report_markdown.py +157 -0
- strop/reporting/run_view.py +118 -0
- strop/services/__init__.py +1 -0
- strop/services/analysis.py +124 -0
- strop/services/clean.py +38 -0
- strop/services/compare.py +60 -0
- strop/services/context.py +70 -0
- strop/services/data.py +65 -0
- strop/services/evaluate.py +105 -0
- strop/services/evaluator.py +169 -0
- strop/services/export.py +90 -0
- strop/services/final_test.py +69 -0
- strop/services/metrics.py +89 -0
- strop/services/optimize.py +187 -0
- strop/services/partitions.py +46 -0
- strop/services/project.py +55 -0
- strop/services/prompts.py +76 -0
- strop/services/report.py +194 -0
- strop/services/review.py +43 -0
- strop/services/scaffold.py +79 -0
- strop/services/turn.py +234 -0
- strop/storage/__init__.py +6 -0
- strop/storage/cache.py +45 -0
- strop/storage/database.py +45 -0
- strop/storage/datasets.py +129 -0
- strop/storage/evaluations.py +160 -0
- strop/storage/migrations/001_initial.sql +5 -0
- strop/storage/migrations/002_datasets.sql +27 -0
- strop/storage/migrations/003_prompt_versions.sql +22 -0
- strop/storage/migrations/004_llm_cache.sql +6 -0
- strop/storage/migrations/005_evaluations.sql +37 -0
- strop/storage/migrations/006_runs.sql +52 -0
- strop/storage/migrations/007_example_feedback.sql +2 -0
- strop/storage/migrations/__init__.py +5 -0
- strop/storage/migrator.py +91 -0
- strop/storage/paths.py +30 -0
- strop/storage/prompts.py +96 -0
- strop/storage/run_models.py +75 -0
- strop/storage/runs.py +263 -0
- strop_cli-0.1.0.dist-info/METADATA +242 -0
- strop_cli-0.1.0.dist-info/RECORD +137 -0
- strop_cli-0.1.0.dist-info/WHEEL +4 -0
- strop_cli-0.1.0.dist-info/entry_points.txt +2 -0
- strop_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
-- One evaluation = one prompt version run over one partition, with per-example results.
|
|
2
|
+
CREATE TABLE evaluations (
|
|
3
|
+
id INTEGER PRIMARY KEY,
|
|
4
|
+
prompt_number INTEGER NOT NULL REFERENCES prompt_versions (number),
|
|
5
|
+
split_id INTEGER NOT NULL REFERENCES splits (id),
|
|
6
|
+
partition TEXT NOT NULL CHECK (partition IN ('train', 'validation', 'test')),
|
|
7
|
+
kind TEXT NOT NULL,
|
|
8
|
+
run_id INTEGER,
|
|
9
|
+
metrics_json TEXT NOT NULL,
|
|
10
|
+
usage_json TEXT NOT NULL,
|
|
11
|
+
details_json TEXT NOT NULL DEFAULT '{}',
|
|
12
|
+
sample_count INTEGER NOT NULL,
|
|
13
|
+
error_count INTEGER NOT NULL,
|
|
14
|
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
15
|
+
);
|
|
16
|
+
|
|
17
|
+
CREATE INDEX evaluations_lookup ON evaluations (prompt_number, partition, kind);
|
|
18
|
+
|
|
19
|
+
CREATE TABLE example_results (
|
|
20
|
+
evaluation_id INTEGER NOT NULL REFERENCES evaluations (id) ON DELETE CASCADE,
|
|
21
|
+
row_id TEXT NOT NULL,
|
|
22
|
+
expected TEXT,
|
|
23
|
+
raw_output TEXT NOT NULL,
|
|
24
|
+
value TEXT,
|
|
25
|
+
parsed_json TEXT,
|
|
26
|
+
error TEXT,
|
|
27
|
+
detail TEXT,
|
|
28
|
+
finish_reason TEXT,
|
|
29
|
+
latency_s REAL NOT NULL DEFAULT 0,
|
|
30
|
+
prompt_tokens INTEGER,
|
|
31
|
+
completion_tokens INTEGER,
|
|
32
|
+
total_tokens INTEGER,
|
|
33
|
+
cost_usd REAL,
|
|
34
|
+
cached INTEGER NOT NULL DEFAULT 0,
|
|
35
|
+
scores_json TEXT NOT NULL DEFAULT '{}',
|
|
36
|
+
PRIMARY KEY (evaluation_id, row_id)
|
|
37
|
+
);
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
-- Optimization runs, their turns, and every candidate prompt that was proposed.
|
|
2
|
+
CREATE TABLE runs (
|
|
3
|
+
id INTEGER PRIMARY KEY,
|
|
4
|
+
status TEXT NOT NULL
|
|
5
|
+
CHECK (status IN ('running', 'paused', 'completed', 'failed', 'interrupted')),
|
|
6
|
+
start_prompt INTEGER NOT NULL REFERENCES prompt_versions (number),
|
|
7
|
+
settings_json TEXT NOT NULL,
|
|
8
|
+
stale_turns INTEGER NOT NULL DEFAULT 0,
|
|
9
|
+
stop_reason TEXT,
|
|
10
|
+
model_calls INTEGER NOT NULL DEFAULT 0,
|
|
11
|
+
cost_usd REAL,
|
|
12
|
+
started_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
13
|
+
finished_at TEXT
|
|
14
|
+
);
|
|
15
|
+
|
|
16
|
+
CREATE TABLE turns (
|
|
17
|
+
run_id INTEGER NOT NULL REFERENCES runs (id) ON DELETE CASCADE,
|
|
18
|
+
number INTEGER NOT NULL,
|
|
19
|
+
status TEXT NOT NULL CHECK (status IN ('running', 'awaiting_approval', 'done')),
|
|
20
|
+
best_before INTEGER NOT NULL REFERENCES prompt_versions (number),
|
|
21
|
+
best_after INTEGER REFERENCES prompt_versions (number),
|
|
22
|
+
train_evaluation_id INTEGER REFERENCES evaluations (id),
|
|
23
|
+
analysis_json TEXT NOT NULL DEFAULT '{}',
|
|
24
|
+
PRIMARY KEY (run_id, number)
|
|
25
|
+
);
|
|
26
|
+
|
|
27
|
+
CREATE TABLE candidates (
|
|
28
|
+
id INTEGER PRIMARY KEY,
|
|
29
|
+
run_id INTEGER NOT NULL REFERENCES runs (id) ON DELETE CASCADE,
|
|
30
|
+
turn INTEGER NOT NULL,
|
|
31
|
+
parent_number INTEGER NOT NULL REFERENCES prompt_versions (number),
|
|
32
|
+
version_number INTEGER REFERENCES prompt_versions (number),
|
|
33
|
+
change_summary TEXT NOT NULL DEFAULT '',
|
|
34
|
+
patterns_json TEXT NOT NULL DEFAULT '[]',
|
|
35
|
+
evidence_json TEXT NOT NULL DEFAULT '[]',
|
|
36
|
+
tradeoffs_json TEXT NOT NULL DEFAULT '[]',
|
|
37
|
+
assumptions_json TEXT NOT NULL DEFAULT '[]',
|
|
38
|
+
raw_response TEXT NOT NULL DEFAULT '',
|
|
39
|
+
evaluation_id INTEGER REFERENCES evaluations (id),
|
|
40
|
+
passes INTEGER,
|
|
41
|
+
primary_delta REAL,
|
|
42
|
+
wins INTEGER,
|
|
43
|
+
losses INTEGER,
|
|
44
|
+
ties INTEGER,
|
|
45
|
+
status TEXT NOT NULL
|
|
46
|
+
CHECK (status IN ('evaluated', 'accepted', 'rejected', 'pending', 'invalid')),
|
|
47
|
+
reason TEXT NOT NULL DEFAULT '',
|
|
48
|
+
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
49
|
+
);
|
|
50
|
+
|
|
51
|
+
CREATE INDEX candidates_by_run ON candidates (run_id, turn);
|
|
52
|
+
CREATE INDEX candidates_by_version ON candidates (version_number);
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""A small versioned SQL migration runner for SQLite."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
import sqlite3
|
|
5
|
+
from collections.abc import Sequence
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from importlib import resources
|
|
8
|
+
|
|
9
|
+
from strop.errors import StorageError
|
|
10
|
+
|
|
11
|
+
MIGRATIONS_PACKAGE = "strop.storage.migrations"
|
|
12
|
+
_FILENAME = re.compile(r"^(?P<version>\d{3,})_(?P<name>[a-z0-9_]+)\.sql$")
|
|
13
|
+
|
|
14
|
+
_CREATE_VERSION_TABLE = """
|
|
15
|
+
CREATE TABLE IF NOT EXISTS schema_version (
|
|
16
|
+
version INTEGER PRIMARY KEY,
|
|
17
|
+
name TEXT NOT NULL,
|
|
18
|
+
applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
19
|
+
)
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class Migration:
|
|
25
|
+
version: int
|
|
26
|
+
name: str
|
|
27
|
+
sql: str
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def discover_migrations(package: str = MIGRATIONS_PACKAGE) -> list[Migration]:
|
|
31
|
+
"""Load migrations shipped inside ``package``, ordered by version."""
|
|
32
|
+
found = []
|
|
33
|
+
for entry in resources.files(package).iterdir():
|
|
34
|
+
match = _FILENAME.match(entry.name)
|
|
35
|
+
if match:
|
|
36
|
+
found.append(
|
|
37
|
+
Migration(int(match["version"]), match["name"], entry.read_text(encoding="utf-8"))
|
|
38
|
+
)
|
|
39
|
+
return _validated(sorted(found, key=lambda m: m.version))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def current_version(conn: sqlite3.Connection) -> int:
|
|
43
|
+
"""The highest applied migration version, or 0 for a new database."""
|
|
44
|
+
conn.execute(_CREATE_VERSION_TABLE)
|
|
45
|
+
row = conn.execute("SELECT MAX(version) FROM schema_version").fetchone()
|
|
46
|
+
return int(row[0] or 0)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def migrate(conn: sqlite3.Connection, migrations: Sequence[Migration] | None = None) -> list[int]:
|
|
50
|
+
"""Apply pending migrations in order and return the versions that were applied.
|
|
51
|
+
|
|
52
|
+
Each migration runs in its own transaction together with its version record,
|
|
53
|
+
so a failure leaves the database at the last good version.
|
|
54
|
+
"""
|
|
55
|
+
available = _validated(list(migrations if migrations is not None else discover_migrations()))
|
|
56
|
+
applied = current_version(conn)
|
|
57
|
+
if available and applied > available[-1].version:
|
|
58
|
+
raise StorageError(
|
|
59
|
+
f"the project database is at schema version {applied}, newer than this "
|
|
60
|
+
f"version of strop understands ({available[-1].version}); upgrade strop"
|
|
61
|
+
)
|
|
62
|
+
done = []
|
|
63
|
+
for migration in (m for m in available if m.version > applied):
|
|
64
|
+
_apply(conn, migration)
|
|
65
|
+
done.append(migration.version)
|
|
66
|
+
return done
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _apply(conn: sqlite3.Connection, migration: Migration) -> None:
|
|
70
|
+
record = (
|
|
71
|
+
f"INSERT INTO schema_version (version, name) "
|
|
72
|
+
f"VALUES ({migration.version}, '{migration.name}');"
|
|
73
|
+
)
|
|
74
|
+
try:
|
|
75
|
+
conn.executescript(f"BEGIN;\n{migration.sql}\n{record}\nCOMMIT;")
|
|
76
|
+
except sqlite3.Error as exc:
|
|
77
|
+
if conn.in_transaction:
|
|
78
|
+
conn.execute("ROLLBACK")
|
|
79
|
+
raise StorageError(
|
|
80
|
+
f"migration {migration.version:03d}_{migration.name} failed: {exc}"
|
|
81
|
+
) from exc
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _validated(migrations: list[Migration]) -> list[Migration]:
|
|
85
|
+
for expected, migration in enumerate(migrations, start=1):
|
|
86
|
+
if migration.version != expected:
|
|
87
|
+
raise StorageError(
|
|
88
|
+
f"migration versions must be consecutive from 1; "
|
|
89
|
+
f"expected {expected:03d}, found {migration.version:03d}"
|
|
90
|
+
)
|
|
91
|
+
return migrations
|
strop/storage/paths.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Where a project keeps its configuration and local state."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
CONFIG_FILENAME = "strop.yaml"
|
|
7
|
+
STATE_DIRNAME = ".strop"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class ProjectPaths:
|
|
12
|
+
"""Locations derived from the project root directory."""
|
|
13
|
+
|
|
14
|
+
root: Path
|
|
15
|
+
|
|
16
|
+
@property
|
|
17
|
+
def config_file(self) -> Path:
|
|
18
|
+
return self.root / CONFIG_FILENAME
|
|
19
|
+
|
|
20
|
+
@property
|
|
21
|
+
def state_dir(self) -> Path:
|
|
22
|
+
return self.root / STATE_DIRNAME
|
|
23
|
+
|
|
24
|
+
@property
|
|
25
|
+
def database(self) -> Path:
|
|
26
|
+
return self.state_dir / "strop.db"
|
|
27
|
+
|
|
28
|
+
def ensure(self) -> None:
|
|
29
|
+
"""Create the state directory if it does not exist."""
|
|
30
|
+
self.state_dir.mkdir(parents=True, exist_ok=True)
|
strop/storage/prompts.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Persistence for prompt versions and the current best pointer."""
|
|
2
|
+
|
|
3
|
+
import sqlite3
|
|
4
|
+
|
|
5
|
+
from strop.errors import ConfigError
|
|
6
|
+
from strop.prompts.models import PromptTemplate
|
|
7
|
+
from strop.prompts.versions import PromptVersion, Source
|
|
8
|
+
from strop.storage.database import transaction
|
|
9
|
+
|
|
10
|
+
BEST_KEY = "best_prompt_version"
|
|
11
|
+
|
|
12
|
+
_COLUMNS = "number, parent_number, content_json, change_summary, reason, source, created_at"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class PromptRepository:
|
|
16
|
+
def __init__(self, conn: sqlite3.Connection) -> None:
|
|
17
|
+
self._conn = conn
|
|
18
|
+
|
|
19
|
+
def add(
|
|
20
|
+
self,
|
|
21
|
+
template: PromptTemplate,
|
|
22
|
+
*,
|
|
23
|
+
parent_number: int | None,
|
|
24
|
+
change_summary: str = "",
|
|
25
|
+
reason: str = "",
|
|
26
|
+
source: Source = "user",
|
|
27
|
+
) -> PromptVersion:
|
|
28
|
+
"""Store a new version with the next number; existing versions are never touched."""
|
|
29
|
+
with transaction(self._conn):
|
|
30
|
+
row = self._conn.execute("SELECT COALESCE(MAX(number), 0) + 1 FROM prompt_versions")
|
|
31
|
+
number = int(row.fetchone()[0])
|
|
32
|
+
self._conn.execute(
|
|
33
|
+
"INSERT INTO prompt_versions (number, parent_number, content_json, content_hash, "
|
|
34
|
+
"change_summary, reason, source) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
35
|
+
(
|
|
36
|
+
number,
|
|
37
|
+
parent_number,
|
|
38
|
+
template.to_json(),
|
|
39
|
+
template.content_hash,
|
|
40
|
+
change_summary,
|
|
41
|
+
reason,
|
|
42
|
+
source,
|
|
43
|
+
),
|
|
44
|
+
)
|
|
45
|
+
return self.get(number)
|
|
46
|
+
|
|
47
|
+
def get(self, number: int) -> PromptVersion:
|
|
48
|
+
row = self._conn.execute(
|
|
49
|
+
f"SELECT {_COLUMNS} FROM prompt_versions WHERE number = ?", (number,)
|
|
50
|
+
).fetchone()
|
|
51
|
+
if row is None:
|
|
52
|
+
raise ConfigError(f"prompt version v{number} does not exist")
|
|
53
|
+
return _to_version(row)
|
|
54
|
+
|
|
55
|
+
def list(self) -> list[PromptVersion]:
|
|
56
|
+
rows = self._conn.execute(f"SELECT {_COLUMNS} FROM prompt_versions ORDER BY number")
|
|
57
|
+
return [_to_version(row) for row in rows.fetchall()]
|
|
58
|
+
|
|
59
|
+
def find_by_hash(self, content_hash: str) -> PromptVersion | None:
|
|
60
|
+
"""The earliest version whose content matches ``content_hash``, if any."""
|
|
61
|
+
row = self._conn.execute(
|
|
62
|
+
f"SELECT {_COLUMNS} FROM prompt_versions WHERE content_hash = ? ORDER BY number",
|
|
63
|
+
(content_hash,),
|
|
64
|
+
).fetchone()
|
|
65
|
+
return _to_version(row) if row else None
|
|
66
|
+
|
|
67
|
+
def first(self) -> PromptVersion | None:
|
|
68
|
+
rows = self.list()
|
|
69
|
+
return rows[0] if rows else None
|
|
70
|
+
|
|
71
|
+
def best_number(self) -> int | None:
|
|
72
|
+
row = self._conn.execute("SELECT value FROM meta WHERE key = ?", (BEST_KEY,)).fetchone()
|
|
73
|
+
return int(row["value"]) if row else None
|
|
74
|
+
|
|
75
|
+
def best(self) -> PromptVersion | None:
|
|
76
|
+
number = self.best_number()
|
|
77
|
+
return self.get(number) if number is not None else None
|
|
78
|
+
|
|
79
|
+
def set_best(self, number: int) -> None:
|
|
80
|
+
"""Point "best" at an existing version. This never deletes or alters any version."""
|
|
81
|
+
self.get(number)
|
|
82
|
+
self._conn.execute(
|
|
83
|
+
"INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", (BEST_KEY, str(number))
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _to_version(row: sqlite3.Row) -> PromptVersion:
|
|
88
|
+
return PromptVersion(
|
|
89
|
+
number=row["number"],
|
|
90
|
+
parent_number=row["parent_number"],
|
|
91
|
+
template=PromptTemplate.from_json(row["content_json"]),
|
|
92
|
+
change_summary=row["change_summary"],
|
|
93
|
+
reason=row["reason"],
|
|
94
|
+
source=row["source"],
|
|
95
|
+
created_at=row["created_at"],
|
|
96
|
+
)
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""Records for optimization runs, turns, and candidates."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
RESUMABLE = ("running", "paused", "interrupted")
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass(frozen=True, slots=True)
|
|
10
|
+
class RunRecord:
|
|
11
|
+
id: int
|
|
12
|
+
status: str
|
|
13
|
+
start_prompt: int
|
|
14
|
+
settings: dict[str, Any]
|
|
15
|
+
stale_turns: int
|
|
16
|
+
stop_reason: str | None
|
|
17
|
+
model_calls: int
|
|
18
|
+
cost_usd: float | None
|
|
19
|
+
started_at: str
|
|
20
|
+
finished_at: str | None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True, slots=True)
|
|
24
|
+
class TurnRecord:
|
|
25
|
+
run_id: int
|
|
26
|
+
number: int
|
|
27
|
+
status: str
|
|
28
|
+
best_before: int
|
|
29
|
+
best_after: int | None
|
|
30
|
+
train_evaluation_id: int | None
|
|
31
|
+
analysis: dict[str, Any]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True, slots=True)
|
|
35
|
+
class NewCandidate:
|
|
36
|
+
run_id: int
|
|
37
|
+
turn: int
|
|
38
|
+
parent_number: int
|
|
39
|
+
status: str
|
|
40
|
+
reason: str = ""
|
|
41
|
+
version_number: int | None = None
|
|
42
|
+
change_summary: str = ""
|
|
43
|
+
patterns: list[str] = field(default_factory=list)
|
|
44
|
+
evidence: list[str] = field(default_factory=list)
|
|
45
|
+
tradeoffs: list[str] = field(default_factory=list)
|
|
46
|
+
assumptions: list[str] = field(default_factory=list)
|
|
47
|
+
raw_response: str = ""
|
|
48
|
+
evaluation_id: int | None = None
|
|
49
|
+
passes: bool | None = None
|
|
50
|
+
primary_delta: float | None = None
|
|
51
|
+
wins: int | None = None
|
|
52
|
+
losses: int | None = None
|
|
53
|
+
ties: int | None = None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass(frozen=True, slots=True)
|
|
57
|
+
class CandidateRecord:
|
|
58
|
+
id: int
|
|
59
|
+
run_id: int
|
|
60
|
+
turn: int
|
|
61
|
+
parent_number: int
|
|
62
|
+
version_number: int | None
|
|
63
|
+
change_summary: str
|
|
64
|
+
patterns: list[str]
|
|
65
|
+
evidence: list[str]
|
|
66
|
+
tradeoffs: list[str]
|
|
67
|
+
assumptions: list[str]
|
|
68
|
+
evaluation_id: int | None
|
|
69
|
+
passes: bool | None
|
|
70
|
+
primary_delta: float | None
|
|
71
|
+
wins: int | None
|
|
72
|
+
losses: int | None
|
|
73
|
+
ties: int | None
|
|
74
|
+
status: str
|
|
75
|
+
reason: str
|
strop/storage/runs.py
ADDED
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
"""Persistence for optimization runs, turns, and candidates."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import sqlite3
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from strop.errors import StorageError
|
|
8
|
+
from strop.storage.run_models import (
|
|
9
|
+
RESUMABLE,
|
|
10
|
+
CandidateRecord,
|
|
11
|
+
NewCandidate,
|
|
12
|
+
RunRecord,
|
|
13
|
+
TurnRecord,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"CandidateRecord",
|
|
18
|
+
"NewCandidate",
|
|
19
|
+
"RunRecord",
|
|
20
|
+
"RunRepository",
|
|
21
|
+
"TurnRecord",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
_CANDIDATE_COLUMNS = (
|
|
25
|
+
"id, run_id, turn, parent_number, version_number, change_summary, patterns_json, "
|
|
26
|
+
"evidence_json, tradeoffs_json, assumptions_json, evaluation_id, passes, primary_delta, "
|
|
27
|
+
"wins, losses, ties, status, reason"
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class RunRepository:
|
|
32
|
+
def __init__(self, conn: sqlite3.Connection) -> None:
|
|
33
|
+
self._conn = conn
|
|
34
|
+
|
|
35
|
+
# -- runs ---------------------------------------------------------------------------
|
|
36
|
+
def create_run(self, start_prompt: int, settings: dict[str, Any]) -> RunRecord:
|
|
37
|
+
cursor = self._conn.execute(
|
|
38
|
+
"INSERT INTO runs (status, start_prompt, settings_json) VALUES ('running', ?, ?)",
|
|
39
|
+
(start_prompt, json.dumps(settings)),
|
|
40
|
+
)
|
|
41
|
+
return self.get_run(int(cursor.lastrowid or 0))
|
|
42
|
+
|
|
43
|
+
def get_run(self, run_id: int) -> RunRecord:
|
|
44
|
+
row = self._conn.execute("SELECT * FROM runs WHERE id = ?", (run_id,)).fetchone()
|
|
45
|
+
if row is None:
|
|
46
|
+
raise StorageError(f"run {run_id} does not exist")
|
|
47
|
+
return _run(row)
|
|
48
|
+
|
|
49
|
+
def latest_run(self) -> RunRecord | None:
|
|
50
|
+
row = self._conn.execute("SELECT * FROM runs ORDER BY id DESC LIMIT 1").fetchone()
|
|
51
|
+
return _run(row) if row else None
|
|
52
|
+
|
|
53
|
+
def all_runs(self) -> list[RunRecord]:
|
|
54
|
+
rows = self._conn.execute("SELECT * FROM runs ORDER BY id").fetchall()
|
|
55
|
+
return [_run(row) for row in rows]
|
|
56
|
+
|
|
57
|
+
def resumable_run(self) -> RunRecord | None:
|
|
58
|
+
marks = ", ".join("?" * len(RESUMABLE))
|
|
59
|
+
row = self._conn.execute(
|
|
60
|
+
f"SELECT * FROM runs WHERE status IN ({marks}) ORDER BY id DESC LIMIT 1", RESUMABLE
|
|
61
|
+
).fetchone()
|
|
62
|
+
return _run(row) if row else None
|
|
63
|
+
|
|
64
|
+
def update_run(
|
|
65
|
+
self,
|
|
66
|
+
run_id: int,
|
|
67
|
+
*,
|
|
68
|
+
status: str | None = None,
|
|
69
|
+
stale_turns: int | None = None,
|
|
70
|
+
stop_reason: str | None = None,
|
|
71
|
+
model_calls: int | None = None,
|
|
72
|
+
cost_usd: float | None = None,
|
|
73
|
+
finished: bool = False,
|
|
74
|
+
) -> None:
|
|
75
|
+
fields = {
|
|
76
|
+
"status": status,
|
|
77
|
+
"stale_turns": stale_turns,
|
|
78
|
+
"stop_reason": stop_reason,
|
|
79
|
+
"model_calls": model_calls,
|
|
80
|
+
"cost_usd": cost_usd,
|
|
81
|
+
}
|
|
82
|
+
updates = {k: v for k, v in fields.items() if v is not None}
|
|
83
|
+
assignments = [f"{key} = ?" for key in updates]
|
|
84
|
+
if finished:
|
|
85
|
+
assignments.append("finished_at = CURRENT_TIMESTAMP")
|
|
86
|
+
if not assignments:
|
|
87
|
+
return
|
|
88
|
+
self._conn.execute(
|
|
89
|
+
f"UPDATE runs SET {', '.join(assignments)} WHERE id = ?", (*updates.values(), run_id)
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
# -- turns --------------------------------------------------------------------------
|
|
93
|
+
def create_turn(self, run_id: int, number: int, best_before: int) -> TurnRecord:
|
|
94
|
+
self._conn.execute(
|
|
95
|
+
"INSERT INTO turns (run_id, number, status, best_before) VALUES (?, ?, 'running', ?)",
|
|
96
|
+
(run_id, number, best_before),
|
|
97
|
+
)
|
|
98
|
+
return self.get_turn(run_id, number)
|
|
99
|
+
|
|
100
|
+
def get_turn(self, run_id: int, number: int) -> TurnRecord:
|
|
101
|
+
row = self._conn.execute(
|
|
102
|
+
"SELECT * FROM turns WHERE run_id = ? AND number = ?", (run_id, number)
|
|
103
|
+
).fetchone()
|
|
104
|
+
if row is None:
|
|
105
|
+
raise StorageError(f"turn {number} of run {run_id} does not exist")
|
|
106
|
+
return _turn(row)
|
|
107
|
+
|
|
108
|
+
def turns(self, run_id: int) -> list[TurnRecord]:
|
|
109
|
+
rows = self._conn.execute(
|
|
110
|
+
"SELECT * FROM turns WHERE run_id = ? ORDER BY number", (run_id,)
|
|
111
|
+
).fetchall()
|
|
112
|
+
return [_turn(row) for row in rows]
|
|
113
|
+
|
|
114
|
+
def update_turn(
|
|
115
|
+
self,
|
|
116
|
+
run_id: int,
|
|
117
|
+
number: int,
|
|
118
|
+
*,
|
|
119
|
+
status: str | None = None,
|
|
120
|
+
best_after: int | None = None,
|
|
121
|
+
train_evaluation_id: int | None = None,
|
|
122
|
+
analysis: dict[str, Any] | None = None,
|
|
123
|
+
) -> None:
|
|
124
|
+
fields: dict[str, Any] = {
|
|
125
|
+
"status": status,
|
|
126
|
+
"best_after": best_after,
|
|
127
|
+
"train_evaluation_id": train_evaluation_id,
|
|
128
|
+
"analysis_json": json.dumps(analysis) if analysis is not None else None,
|
|
129
|
+
}
|
|
130
|
+
updates = {k: v for k, v in fields.items() if v is not None}
|
|
131
|
+
if updates:
|
|
132
|
+
assignments = ", ".join(f"{key} = ?" for key in updates)
|
|
133
|
+
self._conn.execute(
|
|
134
|
+
f"UPDATE turns SET {assignments} WHERE run_id = ? AND number = ?",
|
|
135
|
+
(*updates.values(), run_id, number),
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
# -- candidates ---------------------------------------------------------------------
|
|
139
|
+
def add_candidate(self, new: NewCandidate) -> CandidateRecord:
|
|
140
|
+
cursor = self._conn.execute(
|
|
141
|
+
"INSERT INTO candidates (run_id, turn, parent_number, version_number, change_summary, "
|
|
142
|
+
"patterns_json, evidence_json, tradeoffs_json, assumptions_json, raw_response, "
|
|
143
|
+
"evaluation_id, passes, primary_delta, wins, losses, ties, status, reason) "
|
|
144
|
+
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
145
|
+
(
|
|
146
|
+
new.run_id,
|
|
147
|
+
new.turn,
|
|
148
|
+
new.parent_number,
|
|
149
|
+
new.version_number,
|
|
150
|
+
new.change_summary,
|
|
151
|
+
json.dumps(new.patterns),
|
|
152
|
+
json.dumps(new.evidence),
|
|
153
|
+
json.dumps(new.tradeoffs),
|
|
154
|
+
json.dumps(new.assumptions),
|
|
155
|
+
new.raw_response,
|
|
156
|
+
new.evaluation_id,
|
|
157
|
+
None if new.passes is None else int(new.passes),
|
|
158
|
+
new.primary_delta,
|
|
159
|
+
new.wins,
|
|
160
|
+
new.losses,
|
|
161
|
+
new.ties,
|
|
162
|
+
new.status,
|
|
163
|
+
new.reason,
|
|
164
|
+
),
|
|
165
|
+
)
|
|
166
|
+
return self.get_candidate(int(cursor.lastrowid or 0))
|
|
167
|
+
|
|
168
|
+
def get_candidate(self, candidate_id: int) -> CandidateRecord:
|
|
169
|
+
row = self._conn.execute(
|
|
170
|
+
f"SELECT {_CANDIDATE_COLUMNS} FROM candidates WHERE id = ?", (candidate_id,)
|
|
171
|
+
).fetchone()
|
|
172
|
+
if row is None:
|
|
173
|
+
raise StorageError(f"candidate {candidate_id} does not exist")
|
|
174
|
+
return _candidate(row)
|
|
175
|
+
|
|
176
|
+
def set_candidate_status(self, candidate_id: int, status: str, reason: str) -> None:
|
|
177
|
+
self._conn.execute(
|
|
178
|
+
"UPDATE candidates SET status = ?, reason = ? WHERE id = ?",
|
|
179
|
+
(status, reason, candidate_id),
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
def candidates(self, run_id: int, turn: int | None = None) -> list[CandidateRecord]:
|
|
183
|
+
sql = f"SELECT {_CANDIDATE_COLUMNS} FROM candidates WHERE run_id = ?"
|
|
184
|
+
args: list[Any] = [run_id]
|
|
185
|
+
if turn is not None:
|
|
186
|
+
sql += " AND turn = ?"
|
|
187
|
+
args.append(turn)
|
|
188
|
+
rows = self._conn.execute(sql + " ORDER BY id", args).fetchall()
|
|
189
|
+
return [_candidate(row) for row in rows]
|
|
190
|
+
|
|
191
|
+
def for_version(self, version_number: int) -> CandidateRecord | None:
|
|
192
|
+
row = self._conn.execute(
|
|
193
|
+
f"SELECT {_CANDIDATE_COLUMNS} FROM candidates WHERE version_number = ? "
|
|
194
|
+
"ORDER BY id DESC LIMIT 1",
|
|
195
|
+
(version_number,),
|
|
196
|
+
).fetchone()
|
|
197
|
+
return _candidate(row) if row else None
|
|
198
|
+
|
|
199
|
+
def pending(self) -> list[CandidateRecord]:
|
|
200
|
+
rows = self._conn.execute(
|
|
201
|
+
f"SELECT {_CANDIDATE_COLUMNS} FROM candidates WHERE status = 'pending' ORDER BY id"
|
|
202
|
+
).fetchall()
|
|
203
|
+
return [_candidate(row) for row in rows]
|
|
204
|
+
|
|
205
|
+
def recent(self, limit: int) -> list[CandidateRecord]:
|
|
206
|
+
"""The latest decided candidates across all runs, newest first."""
|
|
207
|
+
rows = self._conn.execute(
|
|
208
|
+
f"SELECT {_CANDIDATE_COLUMNS} FROM candidates WHERE status != 'evaluated' "
|
|
209
|
+
"ORDER BY id DESC LIMIT ?",
|
|
210
|
+
(limit,),
|
|
211
|
+
).fetchall()
|
|
212
|
+
return [_candidate(row) for row in rows]
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _run(row: sqlite3.Row) -> RunRecord:
|
|
216
|
+
return RunRecord(
|
|
217
|
+
row["id"],
|
|
218
|
+
row["status"],
|
|
219
|
+
row["start_prompt"],
|
|
220
|
+
json.loads(row["settings_json"]),
|
|
221
|
+
row["stale_turns"],
|
|
222
|
+
row["stop_reason"],
|
|
223
|
+
row["model_calls"],
|
|
224
|
+
row["cost_usd"],
|
|
225
|
+
row["started_at"],
|
|
226
|
+
row["finished_at"],
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _turn(row: sqlite3.Row) -> TurnRecord:
|
|
231
|
+
return TurnRecord(
|
|
232
|
+
row["run_id"],
|
|
233
|
+
row["number"],
|
|
234
|
+
row["status"],
|
|
235
|
+
row["best_before"],
|
|
236
|
+
row["best_after"],
|
|
237
|
+
row["train_evaluation_id"],
|
|
238
|
+
json.loads(row["analysis_json"]),
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _candidate(row: sqlite3.Row) -> CandidateRecord:
|
|
243
|
+
passes = row["passes"]
|
|
244
|
+
return CandidateRecord(
|
|
245
|
+
row["id"],
|
|
246
|
+
row["run_id"],
|
|
247
|
+
row["turn"],
|
|
248
|
+
row["parent_number"],
|
|
249
|
+
row["version_number"],
|
|
250
|
+
row["change_summary"],
|
|
251
|
+
json.loads(row["patterns_json"]),
|
|
252
|
+
json.loads(row["evidence_json"]),
|
|
253
|
+
json.loads(row["tradeoffs_json"]),
|
|
254
|
+
json.loads(row["assumptions_json"]),
|
|
255
|
+
row["evaluation_id"],
|
|
256
|
+
None if passes is None else bool(passes),
|
|
257
|
+
row["primary_delta"],
|
|
258
|
+
row["wins"],
|
|
259
|
+
row["losses"],
|
|
260
|
+
row["ties"],
|
|
261
|
+
row["status"],
|
|
262
|
+
row["reason"],
|
|
263
|
+
)
|