modelwright 0.1.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- modelwright/__init__.py +148 -0
- modelwright/cli.py +466 -0
- modelwright/conversion.py +931 -0
- modelwright/evaluation.py +173 -0
- modelwright/execution.py +239 -0
- modelwright/extraction.py +662 -0
- modelwright/formulas.py +571 -0
- modelwright/formulas_oracle.py +153 -0
- modelwright/generation.py +726 -0
- modelwright/graph.py +591 -0
- modelwright/oracle_validation.py +59 -0
- modelwright/oracles.py +132 -0
- modelwright/references.py +209 -0
- modelwright/validation.py +475 -0
- modelwright-0.1.0a1.dist-info/METADATA +160 -0
- modelwright-0.1.0a1.dist-info/RECORD +20 -0
- modelwright-0.1.0a1.dist-info/WHEEL +5 -0
- modelwright-0.1.0a1.dist-info/entry_points.txt +2 -0
- modelwright-0.1.0a1.dist-info/licenses/LICENSE +21 -0
- modelwright-0.1.0a1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,475 @@
|
|
|
1
|
+
"""Validation report records.
|
|
2
|
+
|
|
3
|
+
These objects describe validation results; they do not run workbook or generated
|
|
4
|
+
model comparisons themselves.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
from collections.abc import Mapping
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any, Literal
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
JsonValue = str | int | float | bool | None | list[Any] | dict[str, Any]
|
|
17
|
+
OutputKind = Literal["number", "text", "boolean", "blank", "error"]
|
|
18
|
+
ReportStatus = Literal["pass", "fail"]
|
|
19
|
+
DiagnosticSeverity = Literal["info", "warning", "error"]
|
|
20
|
+
TextComparisonMode = Literal["exact"]
|
|
21
|
+
BooleanComparisonMode = Literal["exact"]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class _MissingValue:
|
|
25
|
+
pass
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
MISSING_VALUE = _MissingValue()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class OracleConfig:
|
|
33
|
+
"""Oracle backend configuration from a validation scenario."""
|
|
34
|
+
|
|
35
|
+
backend: str
|
|
36
|
+
options: dict[str, JsonValue] = field(default_factory=dict)
|
|
37
|
+
|
|
38
|
+
@classmethod
|
|
39
|
+
def from_dict(cls, data: dict[str, Any]) -> "OracleConfig":
|
|
40
|
+
backend = data["backend"]
|
|
41
|
+
options = {key: value for key, value in data.items() if key != "backend"}
|
|
42
|
+
return cls(backend=backend, options=options)
|
|
43
|
+
|
|
44
|
+
def to_dict(self) -> dict[str, JsonValue]:
|
|
45
|
+
return {"backend": self.backend, **self.options}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class ScenarioInput:
|
|
50
|
+
"""Input override declared by a validation scenario."""
|
|
51
|
+
|
|
52
|
+
cell_ref: str
|
|
53
|
+
value: JsonValue
|
|
54
|
+
kind: OutputKind
|
|
55
|
+
source: str | None = None
|
|
56
|
+
|
|
57
|
+
@classmethod
|
|
58
|
+
def from_dict(cls, data: dict[str, Any]) -> "ScenarioInput":
|
|
59
|
+
return cls(
|
|
60
|
+
cell_ref=data["cell_ref"],
|
|
61
|
+
value=data.get("value"),
|
|
62
|
+
kind=data["kind"],
|
|
63
|
+
source=data.get("source"),
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
def to_dict(self) -> dict[str, JsonValue]:
|
|
67
|
+
payload: dict[str, JsonValue] = {
|
|
68
|
+
"cell_ref": self.cell_ref,
|
|
69
|
+
"value": self.value,
|
|
70
|
+
"kind": self.kind,
|
|
71
|
+
}
|
|
72
|
+
if self.source is not None:
|
|
73
|
+
payload["source"] = self.source
|
|
74
|
+
return payload
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(frozen=True)
|
|
78
|
+
class ScenarioOutput:
|
|
79
|
+
"""Output expectation declared by a validation scenario."""
|
|
80
|
+
|
|
81
|
+
cell_ref: str
|
|
82
|
+
kind: OutputKind
|
|
83
|
+
tolerance: float | None = None
|
|
84
|
+
|
|
85
|
+
@classmethod
|
|
86
|
+
def from_dict(cls, data: dict[str, Any]) -> "ScenarioOutput":
|
|
87
|
+
return cls(
|
|
88
|
+
cell_ref=data["cell_ref"],
|
|
89
|
+
kind=data["kind"],
|
|
90
|
+
tolerance=data.get("tolerance"),
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
def to_dict(self) -> dict[str, JsonValue]:
|
|
94
|
+
payload: dict[str, JsonValue] = {
|
|
95
|
+
"cell_ref": self.cell_ref,
|
|
96
|
+
"kind": self.kind,
|
|
97
|
+
}
|
|
98
|
+
if self.tolerance is not None:
|
|
99
|
+
payload["tolerance"] = self.tolerance
|
|
100
|
+
return payload
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@dataclass(frozen=True)
|
|
104
|
+
class ComparisonRules:
|
|
105
|
+
"""Default comparison rules declared by a validation scenario."""
|
|
106
|
+
|
|
107
|
+
default_numeric_tolerance: float = 1e-9
|
|
108
|
+
text: TextComparisonMode = "exact"
|
|
109
|
+
boolean: BooleanComparisonMode = "exact"
|
|
110
|
+
|
|
111
|
+
@classmethod
|
|
112
|
+
def from_dict(cls, data: dict[str, Any]) -> "ComparisonRules":
|
|
113
|
+
return cls(
|
|
114
|
+
default_numeric_tolerance=data.get("default_numeric_tolerance", 1e-9),
|
|
115
|
+
text=data.get("text", "exact"),
|
|
116
|
+
boolean=data.get("boolean", "exact"),
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
def to_dict(self) -> dict[str, JsonValue]:
|
|
120
|
+
return {
|
|
121
|
+
"default_numeric_tolerance": self.default_numeric_tolerance,
|
|
122
|
+
"text": self.text,
|
|
123
|
+
"boolean": self.boolean,
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@dataclass(frozen=True)
|
|
128
|
+
class ValidationScenario:
|
|
129
|
+
"""Validation scenario loaded from a JSON boundary."""
|
|
130
|
+
|
|
131
|
+
scenario_id: str
|
|
132
|
+
description: str
|
|
133
|
+
source_workbook: str
|
|
134
|
+
generated_model: str
|
|
135
|
+
oracle: OracleConfig
|
|
136
|
+
inputs: tuple[ScenarioInput, ...]
|
|
137
|
+
outputs: tuple[ScenarioOutput, ...]
|
|
138
|
+
comparison: ComparisonRules
|
|
139
|
+
|
|
140
|
+
@classmethod
|
|
141
|
+
def from_dict(cls, data: dict[str, Any]) -> "ValidationScenario":
|
|
142
|
+
return cls(
|
|
143
|
+
scenario_id=data["scenario_id"],
|
|
144
|
+
description=data.get("description", ""),
|
|
145
|
+
source_workbook=data["source_workbook"],
|
|
146
|
+
generated_model=data["generated_model"],
|
|
147
|
+
oracle=OracleConfig.from_dict(data["oracle"]),
|
|
148
|
+
inputs=tuple(ScenarioInput.from_dict(input_data) for input_data in data.get("inputs", [])),
|
|
149
|
+
outputs=tuple(ScenarioOutput.from_dict(output_data) for output_data in data["outputs"]),
|
|
150
|
+
comparison=ComparisonRules.from_dict(data.get("comparison", {})),
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
def to_dict(self) -> dict[str, JsonValue]:
|
|
154
|
+
return {
|
|
155
|
+
"scenario_id": self.scenario_id,
|
|
156
|
+
"description": self.description,
|
|
157
|
+
"source_workbook": self.source_workbook,
|
|
158
|
+
"generated_model": self.generated_model,
|
|
159
|
+
"oracle": self.oracle.to_dict(),
|
|
160
|
+
"inputs": [scenario_input.to_dict() for scenario_input in self.inputs],
|
|
161
|
+
"outputs": [output.to_dict() for output in self.outputs],
|
|
162
|
+
"comparison": self.comparison.to_dict(),
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def load_validation_scenario(path: str | Path) -> ValidationScenario:
|
|
167
|
+
"""Load a validation scenario JSON file from disk."""
|
|
168
|
+
|
|
169
|
+
scenario_path = Path(path)
|
|
170
|
+
data = json.loads(scenario_path.read_text(encoding="utf-8"))
|
|
171
|
+
return ValidationScenario.from_dict(data)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
@dataclass(frozen=True)
|
|
175
|
+
class Diagnostic:
|
|
176
|
+
"""Run-level diagnostic not tied to one output comparison."""
|
|
177
|
+
|
|
178
|
+
diagnostic_code: str
|
|
179
|
+
message: str
|
|
180
|
+
severity: DiagnosticSeverity = "warning"
|
|
181
|
+
location: str | None = None
|
|
182
|
+
|
|
183
|
+
@classmethod
|
|
184
|
+
def from_dict(cls, data: dict[str, Any]) -> "Diagnostic":
|
|
185
|
+
return cls(
|
|
186
|
+
diagnostic_code=data["diagnostic_code"],
|
|
187
|
+
message=data["message"],
|
|
188
|
+
severity=data.get("severity", "warning"),
|
|
189
|
+
location=data.get("location"),
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
def to_dict(self) -> dict[str, JsonValue]:
|
|
193
|
+
return {
|
|
194
|
+
"diagnostic_code": self.diagnostic_code,
|
|
195
|
+
"message": self.message,
|
|
196
|
+
"severity": self.severity,
|
|
197
|
+
"location": self.location,
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
@dataclass(frozen=True)
|
|
202
|
+
class ComparisonResult:
|
|
203
|
+
"""Comparison result for one declared workbook output."""
|
|
204
|
+
|
|
205
|
+
scenario_id: str
|
|
206
|
+
cell_ref: str
|
|
207
|
+
kind: OutputKind
|
|
208
|
+
generated: JsonValue
|
|
209
|
+
oracle: JsonValue
|
|
210
|
+
matches: bool
|
|
211
|
+
tolerance: float | None
|
|
212
|
+
difference: float | None
|
|
213
|
+
diagnostic_code: str | None
|
|
214
|
+
message: str
|
|
215
|
+
oracle_backend: str
|
|
216
|
+
|
|
217
|
+
@classmethod
|
|
218
|
+
def from_dict(cls, data: dict[str, Any]) -> "ComparisonResult":
|
|
219
|
+
return cls(
|
|
220
|
+
scenario_id=data["scenario_id"],
|
|
221
|
+
cell_ref=data["cell_ref"],
|
|
222
|
+
kind=data["kind"],
|
|
223
|
+
generated=data.get("generated"),
|
|
224
|
+
oracle=data.get("oracle"),
|
|
225
|
+
matches=data["matches"],
|
|
226
|
+
tolerance=data.get("tolerance"),
|
|
227
|
+
difference=data.get("difference"),
|
|
228
|
+
diagnostic_code=data.get("diagnostic_code"),
|
|
229
|
+
message=data["message"],
|
|
230
|
+
oracle_backend=data["oracle_backend"],
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
def to_dict(self) -> dict[str, JsonValue]:
|
|
234
|
+
return {
|
|
235
|
+
"scenario_id": self.scenario_id,
|
|
236
|
+
"cell_ref": self.cell_ref,
|
|
237
|
+
"kind": self.kind,
|
|
238
|
+
"generated": self.generated,
|
|
239
|
+
"oracle": self.oracle,
|
|
240
|
+
"matches": self.matches,
|
|
241
|
+
"tolerance": self.tolerance,
|
|
242
|
+
"difference": self.difference,
|
|
243
|
+
"diagnostic_code": self.diagnostic_code,
|
|
244
|
+
"message": self.message,
|
|
245
|
+
"oracle_backend": self.oracle_backend,
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def compare_scalar_output(
|
|
250
|
+
*,
|
|
251
|
+
scenario_id: str,
|
|
252
|
+
output: ScenarioOutput,
|
|
253
|
+
generated: JsonValue | _MissingValue,
|
|
254
|
+
oracle: JsonValue | _MissingValue,
|
|
255
|
+
oracle_backend: str,
|
|
256
|
+
default_numeric_tolerance: float = 1e-9,
|
|
257
|
+
) -> ComparisonResult:
|
|
258
|
+
"""Compare one observed generated value against one oracle value."""
|
|
259
|
+
|
|
260
|
+
if generated is MISSING_VALUE and oracle is MISSING_VALUE:
|
|
261
|
+
return ComparisonResult(
|
|
262
|
+
scenario_id=scenario_id,
|
|
263
|
+
cell_ref=output.cell_ref,
|
|
264
|
+
kind=output.kind,
|
|
265
|
+
generated=None,
|
|
266
|
+
oracle=None,
|
|
267
|
+
matches=False,
|
|
268
|
+
tolerance=_tolerance_for(output, default_numeric_tolerance),
|
|
269
|
+
difference=None,
|
|
270
|
+
diagnostic_code="missing_generated_and_oracle_output",
|
|
271
|
+
message="generated and oracle outputs are missing",
|
|
272
|
+
oracle_backend=oracle_backend,
|
|
273
|
+
)
|
|
274
|
+
if generated is MISSING_VALUE:
|
|
275
|
+
return ComparisonResult(
|
|
276
|
+
scenario_id=scenario_id,
|
|
277
|
+
cell_ref=output.cell_ref,
|
|
278
|
+
kind=output.kind,
|
|
279
|
+
generated=None,
|
|
280
|
+
oracle=oracle,
|
|
281
|
+
matches=False,
|
|
282
|
+
tolerance=_tolerance_for(output, default_numeric_tolerance),
|
|
283
|
+
difference=None,
|
|
284
|
+
diagnostic_code="missing_generated_output",
|
|
285
|
+
message="generated output is missing",
|
|
286
|
+
oracle_backend=oracle_backend,
|
|
287
|
+
)
|
|
288
|
+
if oracle is MISSING_VALUE:
|
|
289
|
+
return ComparisonResult(
|
|
290
|
+
scenario_id=scenario_id,
|
|
291
|
+
cell_ref=output.cell_ref,
|
|
292
|
+
kind=output.kind,
|
|
293
|
+
generated=generated,
|
|
294
|
+
oracle=None,
|
|
295
|
+
matches=False,
|
|
296
|
+
tolerance=_tolerance_for(output, default_numeric_tolerance),
|
|
297
|
+
difference=None,
|
|
298
|
+
diagnostic_code="missing_oracle_output",
|
|
299
|
+
message="oracle output is missing",
|
|
300
|
+
oracle_backend=oracle_backend,
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
if output.kind == "number":
|
|
304
|
+
return _compare_number(
|
|
305
|
+
scenario_id=scenario_id,
|
|
306
|
+
output=output,
|
|
307
|
+
generated=generated,
|
|
308
|
+
oracle=oracle,
|
|
309
|
+
oracle_backend=oracle_backend,
|
|
310
|
+
default_numeric_tolerance=default_numeric_tolerance,
|
|
311
|
+
)
|
|
312
|
+
if output.kind == "text":
|
|
313
|
+
return _compare_text(
|
|
314
|
+
scenario_id=scenario_id,
|
|
315
|
+
output=output,
|
|
316
|
+
generated=generated,
|
|
317
|
+
oracle=oracle,
|
|
318
|
+
oracle_backend=oracle_backend,
|
|
319
|
+
)
|
|
320
|
+
|
|
321
|
+
return ComparisonResult(
|
|
322
|
+
scenario_id=scenario_id,
|
|
323
|
+
cell_ref=output.cell_ref,
|
|
324
|
+
kind=output.kind,
|
|
325
|
+
generated=generated,
|
|
326
|
+
oracle=oracle,
|
|
327
|
+
matches=False,
|
|
328
|
+
tolerance=_tolerance_for(output, default_numeric_tolerance),
|
|
329
|
+
difference=None,
|
|
330
|
+
diagnostic_code="unsupported_output_kind",
|
|
331
|
+
message=f"unsupported output kind: {output.kind}",
|
|
332
|
+
oracle_backend=oracle_backend,
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _tolerance_for(output: ScenarioOutput, default_numeric_tolerance: float) -> float | None:
|
|
337
|
+
if output.kind != "number":
|
|
338
|
+
return None
|
|
339
|
+
return output.tolerance if output.tolerance is not None else default_numeric_tolerance
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _compare_number(
|
|
343
|
+
*,
|
|
344
|
+
scenario_id: str,
|
|
345
|
+
output: ScenarioOutput,
|
|
346
|
+
generated: JsonValue,
|
|
347
|
+
oracle: JsonValue,
|
|
348
|
+
oracle_backend: str,
|
|
349
|
+
default_numeric_tolerance: float,
|
|
350
|
+
) -> ComparisonResult:
|
|
351
|
+
tolerance = _tolerance_for(output, default_numeric_tolerance)
|
|
352
|
+
if not _is_number(generated) or not _is_number(oracle):
|
|
353
|
+
return ComparisonResult(
|
|
354
|
+
scenario_id=scenario_id,
|
|
355
|
+
cell_ref=output.cell_ref,
|
|
356
|
+
kind=output.kind,
|
|
357
|
+
generated=generated,
|
|
358
|
+
oracle=oracle,
|
|
359
|
+
matches=False,
|
|
360
|
+
tolerance=tolerance,
|
|
361
|
+
difference=None,
|
|
362
|
+
diagnostic_code="numeric_type_mismatch",
|
|
363
|
+
message="generated and oracle values must both be numeric",
|
|
364
|
+
oracle_backend=oracle_backend,
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
difference = abs(float(generated) - float(oracle))
|
|
368
|
+
matches = difference <= float(tolerance)
|
|
369
|
+
return ComparisonResult(
|
|
370
|
+
scenario_id=scenario_id,
|
|
371
|
+
cell_ref=output.cell_ref,
|
|
372
|
+
kind=output.kind,
|
|
373
|
+
generated=generated,
|
|
374
|
+
oracle=oracle,
|
|
375
|
+
matches=matches,
|
|
376
|
+
tolerance=tolerance,
|
|
377
|
+
difference=difference,
|
|
378
|
+
diagnostic_code=None if matches else "numeric_mismatch",
|
|
379
|
+
message="values match" if matches else "generated value differs from oracle value",
|
|
380
|
+
oracle_backend=oracle_backend,
|
|
381
|
+
)
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _compare_text(
|
|
385
|
+
*,
|
|
386
|
+
scenario_id: str,
|
|
387
|
+
output: ScenarioOutput,
|
|
388
|
+
generated: JsonValue,
|
|
389
|
+
oracle: JsonValue,
|
|
390
|
+
oracle_backend: str,
|
|
391
|
+
) -> ComparisonResult:
|
|
392
|
+
matches = isinstance(generated, str) and isinstance(oracle, str) and generated == oracle
|
|
393
|
+
return ComparisonResult(
|
|
394
|
+
scenario_id=scenario_id,
|
|
395
|
+
cell_ref=output.cell_ref,
|
|
396
|
+
kind=output.kind,
|
|
397
|
+
generated=generated,
|
|
398
|
+
oracle=oracle,
|
|
399
|
+
matches=matches,
|
|
400
|
+
tolerance=None,
|
|
401
|
+
difference=None,
|
|
402
|
+
diagnostic_code=None if matches else "text_mismatch",
|
|
403
|
+
message="values match" if matches else "generated text differs from oracle text",
|
|
404
|
+
oracle_backend=oracle_backend,
|
|
405
|
+
)
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _is_number(value: JsonValue) -> bool:
|
|
409
|
+
return isinstance(value, int | float) and not isinstance(value, bool)
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
@dataclass(frozen=True)
|
|
413
|
+
class ValidationReport:
|
|
414
|
+
"""Validation report for one scenario."""
|
|
415
|
+
|
|
416
|
+
scenario_id: str
|
|
417
|
+
oracle_backend: str
|
|
418
|
+
comparisons: tuple[ComparisonResult, ...] = field(default_factory=tuple)
|
|
419
|
+
diagnostics: tuple[Diagnostic, ...] = field(default_factory=tuple)
|
|
420
|
+
|
|
421
|
+
@property
|
|
422
|
+
def mismatches(self) -> tuple[ComparisonResult, ...]:
|
|
423
|
+
return tuple(comparison for comparison in self.comparisons if not comparison.matches)
|
|
424
|
+
|
|
425
|
+
@property
|
|
426
|
+
def status(self) -> ReportStatus:
|
|
427
|
+
if self.mismatches or any(diagnostic.severity == "error" for diagnostic in self.diagnostics):
|
|
428
|
+
return "fail"
|
|
429
|
+
return "pass"
|
|
430
|
+
|
|
431
|
+
@classmethod
|
|
432
|
+
def from_dict(cls, data: dict[str, Any]) -> "ValidationReport":
|
|
433
|
+
return cls(
|
|
434
|
+
scenario_id=data["scenario_id"],
|
|
435
|
+
oracle_backend=data["oracle_backend"],
|
|
436
|
+
comparisons=tuple(ComparisonResult.from_dict(item) for item in data.get("comparisons", [])),
|
|
437
|
+
diagnostics=tuple(Diagnostic.from_dict(item) for item in data.get("diagnostics", [])),
|
|
438
|
+
)
|
|
439
|
+
|
|
440
|
+
def to_dict(self) -> dict[str, JsonValue]:
|
|
441
|
+
return {
|
|
442
|
+
"scenario_id": self.scenario_id,
|
|
443
|
+
"oracle_backend": self.oracle_backend,
|
|
444
|
+
"status": self.status,
|
|
445
|
+
"comparisons": [comparison.to_dict() for comparison in self.comparisons],
|
|
446
|
+
"mismatches": [comparison.to_dict() for comparison in self.mismatches],
|
|
447
|
+
"diagnostics": [diagnostic.to_dict() for diagnostic in self.diagnostics],
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def build_validation_report(
|
|
452
|
+
*,
|
|
453
|
+
scenario: ValidationScenario,
|
|
454
|
+
generated_values: Mapping[str, JsonValue],
|
|
455
|
+
oracle_values: Mapping[str, JsonValue],
|
|
456
|
+
) -> ValidationReport:
|
|
457
|
+
"""Build a report from scenario outputs and already-observed values."""
|
|
458
|
+
|
|
459
|
+
comparisons = tuple(
|
|
460
|
+
compare_scalar_output(
|
|
461
|
+
scenario_id=scenario.scenario_id,
|
|
462
|
+
output=output,
|
|
463
|
+
generated=generated_values.get(output.cell_ref, MISSING_VALUE),
|
|
464
|
+
oracle=oracle_values.get(output.cell_ref, MISSING_VALUE),
|
|
465
|
+
oracle_backend=scenario.oracle.backend,
|
|
466
|
+
default_numeric_tolerance=scenario.comparison.default_numeric_tolerance,
|
|
467
|
+
)
|
|
468
|
+
for output in scenario.outputs
|
|
469
|
+
)
|
|
470
|
+
|
|
471
|
+
return ValidationReport(
|
|
472
|
+
scenario_id=scenario.scenario_id,
|
|
473
|
+
oracle_backend=scenario.oracle.backend,
|
|
474
|
+
comparisons=comparisons,
|
|
475
|
+
)
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: modelwright
|
|
3
|
+
Version: 0.1.0a1
|
|
4
|
+
Summary: Tools for converting spreadsheet workbooks into transparent Python models.
|
|
5
|
+
Author: UBC FRESH Lab
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/UBC-FRESH/modelwright
|
|
8
|
+
Project-URL: Documentation, https://ubc-fresh.github.io/modelwright/
|
|
9
|
+
Project-URL: Repository, https://github.com/UBC-FRESH/modelwright
|
|
10
|
+
Project-URL: Issues, https://github.com/UBC-FRESH/modelwright/issues
|
|
11
|
+
Project-URL: Changelog, https://github.com/UBC-FRESH/modelwright/blob/main/CHANGE_LOG.md
|
|
12
|
+
Keywords: spreadsheets,excel,model-conversion,reproducible-models,validation
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Topic :: Office/Business
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Requires-Dist: openpyxl>=3.1
|
|
27
|
+
Requires-Dist: rich>=13
|
|
28
|
+
Requires-Dist: typer>=0.9
|
|
29
|
+
Provides-Extra: docs
|
|
30
|
+
Requires-Dist: sphinx>=7; extra == "docs"
|
|
31
|
+
Requires-Dist: sphinx-rtd-theme>=2; extra == "docs"
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
34
|
+
Requires-Dist: formulas; extra == "dev"
|
|
35
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
36
|
+
Requires-Dist: ruff>=0.8; extra == "dev"
|
|
37
|
+
Requires-Dist: sphinx>=7; extra == "dev"
|
|
38
|
+
Requires-Dist: sphinx-rtd-theme>=2; extra == "dev"
|
|
39
|
+
Requires-Dist: twine>=5; extra == "dev"
|
|
40
|
+
Provides-Extra: oracle
|
|
41
|
+
Requires-Dist: formulas; extra == "oracle"
|
|
42
|
+
Provides-Extra: quality
|
|
43
|
+
Requires-Dist: ruff>=0.8; extra == "quality"
|
|
44
|
+
Provides-Extra: release
|
|
45
|
+
Requires-Dist: build>=1.2; extra == "release"
|
|
46
|
+
Requires-Dist: twine>=5; extra == "release"
|
|
47
|
+
Provides-Extra: test
|
|
48
|
+
Requires-Dist: formulas; extra == "test"
|
|
49
|
+
Requires-Dist: pytest>=8; extra == "test"
|
|
50
|
+
Dynamic: license-file
|
|
51
|
+
|
|
52
|
+
# Modelwright
|
|
53
|
+
|
|
54
|
+
`modelwright` is an early-stage project for turning spreadsheet workbooks into transparent, version-controlled, standalone Python models.
|
|
55
|
+
|
|
56
|
+
The intended direction is a generic workflow that can inspect workbook structure, extract formulas and dependencies, generate maintainable Python source, and validate the generated model against the original workbook outputs.
|
|
57
|
+
|
|
58
|
+
This repository is currently an early implementation skeleton. It defines minimal Python package and test scaffolding plus initial validation, extraction, graph, generation, oracle records, and thin JSON command-line wrappers, but does not yet provide a release stability guarantee, catalog schema, or full workbook conversion.
|
|
59
|
+
|
|
60
|
+
## Current Focus
|
|
61
|
+
|
|
62
|
+
- Build the first package-backed validation/report, workbook extraction, generation, and CLI cores.
|
|
63
|
+
- Keep extraction, code generation, validation, diagnostics, and reporting responsibilities separate.
|
|
64
|
+
- Avoid committing private notes, source workbooks, generated clones, or large artifacts while the project shape is still being established.
|
|
65
|
+
|
|
66
|
+
## Python API Boundary
|
|
67
|
+
|
|
68
|
+
The durable API is organized by module responsibility:
|
|
69
|
+
|
|
70
|
+
- `modelwright.extraction`: workbook extraction records and `extract_workbook`.
|
|
71
|
+
- `modelwright.graph`: dependency graph records and `build_dependency_graph`.
|
|
72
|
+
- `modelwright.formulas`: formula expression records, translation helpers, and reference-index helpers.
|
|
73
|
+
- `modelwright.generation`: generated-module records and `generate_python_module`.
|
|
74
|
+
- `modelwright.validation`: validation scenarios, scalar comparisons, and report records.
|
|
75
|
+
- `modelwright.oracles`, `modelwright.formulas_oracle`, and `modelwright.oracle_validation`: oracle request/result records, optional `formulas` oracle execution, and oracle-backed report assembly.
|
|
76
|
+
|
|
77
|
+
The package root `modelwright` exposes a curated convenience facade for those records and functions. Module-level imports remain preferred for implementation work because this project is still pre-release.
|
|
78
|
+
|
|
79
|
+
## Command-Line Interface
|
|
80
|
+
|
|
81
|
+
Bootstrap the repo-local virtual environment before using the console script:
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
scripts/bootstrap_dev_env.sh
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
The current CLI prints JSON to stdout and stays close to the Python APIs:
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
modelwright workbook extract path/to/workbook.xlsx > tmp/extraction.json
|
|
91
|
+
modelwright workbook graph path/to/workbook.xlsx > tmp/dependency-graph.json
|
|
92
|
+
modelwright conversion plan path/to/workbook.xlsx > tmp/conversion-plan.json
|
|
93
|
+
modelwright model generate --contract tmp/contract.json --expressions tmp/expressions.json --constants tmp/constants.json --out tmp/generated_model.py > tmp/generation-result.json
|
|
94
|
+
modelwright validation report --scenario tests/fixtures/synthetic_model/baseline_scenario.json --generated-values tmp/generated-values.json --oracle-values tmp/oracle-values.json > tmp/validation-report.json
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
These commands do not provide a one-step workbook converter. `conversion plan` reports extraction, graphing, formula-translation, and residual-blocker status; `model generate` expects explicit generated-module and formula-expression JSON inputs; and `validation report` compares already-observed generated/oracle values. See `planning/cli-json-workflows.md` for JSON examples and workflow boundaries.
|
|
98
|
+
|
|
99
|
+
## Local Development
|
|
100
|
+
|
|
101
|
+
Bootstrap a repo-local virtual environment:
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
scripts/bootstrap_dev_env.sh
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
This installs Modelwright with the `dev` extra:
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
.venv/bin/python -m pip install -e '.[dev]'
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Run lint checks:
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
.venv/bin/python -m ruff check .
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Run tests:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
.venv/bin/python -m pytest
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Build docs locally:
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
.venv/bin/sphinx-build -b html docs _build/html -W
|
|
129
|
+
.venv/bin/python scripts/verify_docs_theme.py _build/html
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Restore the public external FABLE benchmark workbooks into ignored local paths:
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
scripts/bootstrap_dev_env.sh --benchmarks
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
`modelwright` is pre-release. The first planned alpha line is `0.1.0a1`; alpha releases must not be described as full-workbook conversion guarantees.
|
|
139
|
+
|
|
140
|
+
Check release artifacts locally:
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
scripts/check_release_artifacts.sh
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
Release checks write build outputs under ignored `tmp/release-checks/`.
|
|
147
|
+
|
|
148
|
+
See `docs/guides/release-deployment.rst` for the release and deployment runbook.
|
|
149
|
+
|
|
150
|
+
## Repository Conventions
|
|
151
|
+
|
|
152
|
+
- `AGENTS.md` is the working contract for AI coding agents.
|
|
153
|
+
- `CONTRIBUTING.md` is the contributor onboarding and development workflow guide.
|
|
154
|
+
- `ROADMAP.md` is the current plan and next-step tracker.
|
|
155
|
+
- `CHANGE_LOG.md` is the append-only project narrative.
|
|
156
|
+
- `planning/` contains focused design notes and research records that are too detailed for the roadmap.
|
|
157
|
+
- `benchmarks/` contains tracked metadata for official external benchmarks; large workbook binaries remain untracked and are restored locally under `tmp/`.
|
|
158
|
+
- `src/modelwright/` contains the importable Python package.
|
|
159
|
+
- `tests/` contains package-backed tests and tracked synthetic fixture helpers.
|
|
160
|
+
- `tmp/` is ignored local working space for private notes, source workbooks, experiments, and generated scratch outputs.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
modelwright/__init__.py,sha256=niukW7pkRKSkkC77iOEqt1gzT9rYaXXh8UfSXUvyibc,3573
|
|
2
|
+
modelwright/cli.py,sha256=MFyn0iRm6NGIRZevuV2gpQ7g7V1iU0nGelctzFlWgAE,13891
|
|
3
|
+
modelwright/conversion.py,sha256=dbVLrDSnJp9Ffi0HwwVr7IK14xcF0OAFfVNYm7855_0,35788
|
|
4
|
+
modelwright/evaluation.py,sha256=awwePtzODiA4dyjbh2-bS3LMY3qclxt9G7L2CIjOw4Y,6476
|
|
5
|
+
modelwright/execution.py,sha256=JnoFGoGdLiEw4BlZdwzqc4GNbs-oFB3C6gdmB8ZsYhA,8087
|
|
6
|
+
modelwright/extraction.py,sha256=3_EI4dlgoRRP-TtjbNHtfyEF01UNFAaSpxyHiEb9p9o,22576
|
|
7
|
+
modelwright/formulas.py,sha256=rGuZCsdT0qCjG_dnYpqB4DHwAc-cCSgBk2JmyvjGidw,21259
|
|
8
|
+
modelwright/formulas_oracle.py,sha256=2GAk_oQDLFzmhDqmv2soTOcORUwgahm5XulUDbxAn_U,5167
|
|
9
|
+
modelwright/generation.py,sha256=7DMB5MVnX6gg78iWRMR90Z7o3F1_-tRknNSiKxydOB4,28979
|
|
10
|
+
modelwright/graph.py,sha256=BX6IyI5A52RBjm4uTSBW9hfFZmJIqzwU-dn9qDTlrFg,20112
|
|
11
|
+
modelwright/oracle_validation.py,sha256=drvFik9x7J2PtJBLekccsFMEIkQisdgoRc05Kz57qkA,1738
|
|
12
|
+
modelwright/oracles.py,sha256=WjiVFK0Kg5VpILEIoC2lQlqJzN-5MZGUB967qWW34K4,4478
|
|
13
|
+
modelwright/references.py,sha256=ur2n_jeQ5H6Tyz31vwEYSmWaQXA57C6uD2YOhhf7aG8,6593
|
|
14
|
+
modelwright/validation.py,sha256=_Oo2cAHTUGPdEAdw60Jec7b4xW88AY1iev0coIIcnRw,15320
|
|
15
|
+
modelwright-0.1.0a1.dist-info/licenses/LICENSE,sha256=-wxpwq3CXqCZyuDvzOEdo6nxxy1Rz1Mcudjnq6uBQ3E,1070
|
|
16
|
+
modelwright-0.1.0a1.dist-info/METADATA,sha256=hMQewpYCLTjaHdBjx3aAK85rYkywO58BeNfqFtj7YA8,7117
|
|
17
|
+
modelwright-0.1.0a1.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
18
|
+
modelwright-0.1.0a1.dist-info/entry_points.txt,sha256=SM5N6V17pwDqPMaRL4kY6zOdfkIDxoTl5OH3nFtu2SI,52
|
|
19
|
+
modelwright-0.1.0a1.dist-info/top_level.txt,sha256=NZNmVeA6iHFviNzQD-6DaW6tGDUcgXBoPTW4CcjM43k,12
|
|
20
|
+
modelwright-0.1.0a1.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 UBC FRESH Lab
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
modelwright
|