backspec 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- backspec/__init__.py +16 -0
- backspec/host/__init__.py +0 -0
- backspec/host/adapters/__init__.py +0 -0
- backspec/host/adapters/cache.py +103 -0
- backspec/host/adapters/cache_admin.py +64 -0
- backspec/host/adapters/capabilities_file.py +39 -0
- backspec/host/adapters/claude_code.py +77 -0
- backspec/host/adapters/codex.py +67 -0
- backspec/host/adapters/fake.py +39 -0
- backspec/host/adapters/filesystem.py +50 -0
- backspec/host/adapters/review_yaml.py +33 -0
- backspec/host/adapters/treesitter.py +130 -0
- backspec/host/cli.py +313 -0
- backspec/host/composition.py +211 -0
- backspec/kernel/__init__.py +0 -0
- backspec/kernel/llm.py +64 -0
- backspec/kernel/prompts.py +10 -0
- backspec/kernel/templates.py +12 -0
- backspec/kernel/text.py +23 -0
- backspec/kernel/usage.py +50 -0
- backspec/modules/__init__.py +0 -0
- backspec/modules/capabilities/__init__.py +0 -0
- backspec/modules/capabilities/domain.py +108 -0
- backspec/modules/capabilities/facade.py +48 -0
- backspec/modules/capabilities/ports.py +21 -0
- backspec/modules/capabilities/use_cases.py +63 -0
- backspec/modules/discovery/__init__.py +0 -0
- backspec/modules/discovery/domain.py +87 -0
- backspec/modules/discovery/facade.py +16 -0
- backspec/modules/discovery/ports.py +14 -0
- backspec/modules/discovery/use_cases.py +19 -0
- backspec/modules/evidence/__init__.py +0 -0
- backspec/modules/evidence/domain.py +276 -0
- backspec/modules/evidence/facade.py +24 -0
- backspec/modules/evidence/ports.py +12 -0
- backspec/modules/evidence/use_cases.py +128 -0
- backspec/modules/extraction/__init__.py +0 -0
- backspec/modules/extraction/facade.py +6 -0
- backspec/modules/extraction/use_cases.py +228 -0
- backspec/modules/lint/__init__.py +0 -0
- backspec/modules/lint/domain.py +152 -0
- backspec/modules/lint/facade.py +7 -0
- backspec/modules/lint/use_cases.py +16 -0
- backspec/modules/report/__init__.py +0 -0
- backspec/modules/report/domain.py +128 -0
- backspec/modules/report/dto.py +92 -0
- backspec/modules/report/facade.py +11 -0
- backspec/modules/report/renderers.py +220 -0
- backspec/modules/report/use_cases.py +127 -0
- backspec/modules/requirements/__init__.py +0 -0
- backspec/modules/requirements/domain.py +272 -0
- backspec/modules/requirements/facade.py +26 -0
- backspec/modules/requirements/use_cases.py +135 -0
- backspec/modules/review/__init__.py +0 -0
- backspec/modules/review/domain.py +129 -0
- backspec/modules/review/facade.py +32 -0
- backspec/modules/review/ports.py +15 -0
- backspec/modules/review/use_cases.py +89 -0
- backspec/modules/staleness/__init__.py +0 -0
- backspec/modules/staleness/domain.py +75 -0
- backspec/modules/staleness/facade.py +17 -0
- backspec/modules/staleness/ports.py +7 -0
- backspec/modules/staleness/use_cases.py +39 -0
- backspec/modules/triage/__init__.py +0 -0
- backspec/modules/triage/domain.py +144 -0
- backspec/modules/triage/facade.py +24 -0
- backspec/modules/triage/use_cases.py +79 -0
- backspec/prompts/assess.md +32 -0
- backspec/prompts/assign.md +18 -0
- backspec/prompts/consolidate.md +52 -0
- backspec/prompts/observe.md +52 -0
- backspec/prompts/orient.md +33 -0
- backspec/prompts/synthesise.md +97 -0
- backspec/prompts/triage.md +34 -0
- backspec/py.typed +0 -0
- backspec/templates/libre-baskerville-400.woff2 +0 -0
- backspec/templates/libre-baskerville-700.woff2 +0 -0
- backspec/templates/libre-baskerville-OFL.txt +93 -0
- backspec/templates/report.html +336 -0
- backspec-0.0.1.dist-info/METADATA +430 -0
- backspec-0.0.1.dist-info/RECORD +85 -0
- backspec-0.0.1.dist-info/WHEEL +4 -0
- backspec-0.0.1.dist-info/entry_points.txt +2 -0
- backspec-0.0.1.dist-info/licenses/LICENSE +21 -0
- backspec-0.0.1.dist-info/licenses/src/backspec/templates/libre-baskerville-OFL.txt +93 -0
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
"""The renderers: pure functions from a result to text. They write no files; the host does.
|
|
2
|
+
|
|
3
|
+
To add a format, write `def render_x(result, **options) -> RenderOutput`, add it to RENDERERS,
|
|
4
|
+
and give it a line in the README. A renderer takes every option the command line can pass and
|
|
5
|
+
ignores the ones that do not concern it.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import base64
|
|
10
|
+
import collections
|
|
11
|
+
import csv
|
|
12
|
+
import io
|
|
13
|
+
import json
|
|
14
|
+
import re
|
|
15
|
+
from pathlib import PurePath
|
|
16
|
+
from typing import Callable
|
|
17
|
+
|
|
18
|
+
from backspec.kernel import templates
|
|
19
|
+
from backspec.modules.report.domain import RenderOutput, one_line, requirement_title, with_shall
|
|
20
|
+
from backspec.modules.report.dto import ExtractionResult
|
|
21
|
+
from backspec.modules.report.use_cases import to_markdown
|
|
22
|
+
from backspec.modules.requirements.facade import Requirement
|
|
23
|
+
|
|
24
|
+
Renderer = Callable[..., RenderOutput]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class UnknownFormat(ValueError):
|
|
28
|
+
pass
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def render_json(result: ExtractionResult, **options) -> RenderOutput:
|
|
32
|
+
return RenderOutput(text=result.to_json())
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def render_markdown(result: ExtractionResult, include_technical: bool = False, **options) -> RenderOutput:
|
|
36
|
+
return RenderOutput(text=to_markdown(result, include_technical=include_technical))
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
CSV_COLUMNS = ["requirement_id", "capability", "statement", "conditions", "level", "support", "review", "scope",
|
|
40
|
+
"evidence_role", "path", "first_line", "last_line", "quote"]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def render_csv(result: ExtractionResult, include_hidden: bool = False, **options) -> RenderOutput:
|
|
44
|
+
"""One row per requirement and evidence pair. Hidden requirements only on request, marked in
|
|
45
|
+
a `hidden` column."""
|
|
46
|
+
columns = [*CSV_COLUMNS, "hidden"] if include_hidden else CSV_COLUMNS
|
|
47
|
+
buffer = io.StringIO()
|
|
48
|
+
writer = csv.writer(buffer, lineterminator="\n")
|
|
49
|
+
writer.writerow(columns)
|
|
50
|
+
rows = [(r, False) for r in result.requirements]
|
|
51
|
+
if include_hidden:
|
|
52
|
+
rows += [(r, True) for r in result.hidden]
|
|
53
|
+
for r, hidden in rows:
|
|
54
|
+
for e in r.evidence:
|
|
55
|
+
row = [r.id, r.capability, r.statement, " | ".join(r.conditions), r.level, r.support, r.review, r.scope,
|
|
56
|
+
e.role, e.path, e.lines[0], e.lines[1], e.quote]
|
|
57
|
+
writer.writerow([*row, "true" if hidden else "false"] if include_hidden else row)
|
|
58
|
+
return RenderOutput(text=buffer.getvalue())
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
EXPORTED_SUPPORT = ("direct", "partial")
|
|
62
|
+
NO_SCENARIO = "no evidenced scenario"
|
|
63
|
+
NO_SCENARIO_MARKER = "> No scenario: backspec found no test or example that shows this case."
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _evidence_line(r: Requirement) -> str:
|
|
67
|
+
cites = dict.fromkeys(f"{e.path}:{e.lines[0]}-{e.lines[1]}" for e in r.evidence)
|
|
68
|
+
return "Evidence: " + ", ".join(cites)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _not_exported_reason(r: Requirement, strict: bool) -> str | None:
|
|
72
|
+
if r.support not in EXPORTED_SUPPORT:
|
|
73
|
+
what = "was not assessed" if r.support == "not_assessed" else f"is {r.support}"
|
|
74
|
+
return f"support {what}" + (f": {one_line(r.support_note)}" if r.support_note else "")
|
|
75
|
+
if strict and not r.scenarios:
|
|
76
|
+
return NO_SCENARIO
|
|
77
|
+
return None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _spec_file(name: str, summary: str, requirements: list[Requirement]) -> str:
|
|
81
|
+
out = [f"# {one_line(name)} Specification", "", "## Purpose", "", one_line(summary) or one_line(name), "",
|
|
82
|
+
"## Requirements", ""]
|
|
83
|
+
titles: collections.Counter[str] = collections.Counter()
|
|
84
|
+
for r in requirements:
|
|
85
|
+
title = requirement_title(r.statement)
|
|
86
|
+
titles[title.lower()] += 1
|
|
87
|
+
if titles[title.lower()] > 1: # a spec keeps one block per title
|
|
88
|
+
title = f"{title} ({titles[title.lower()]})"
|
|
89
|
+
out += [f"### Requirement: {title}", "", with_shall(r.statement), "", _evidence_line(r), ""]
|
|
90
|
+
if not r.scenarios:
|
|
91
|
+
out += [NO_SCENARIO_MARKER, ""]
|
|
92
|
+
for s in r.scenarios:
|
|
93
|
+
out += [f"#### Scenario: {one_line(s.name)}", f"- **GIVEN** {one_line(s.given)}",
|
|
94
|
+
f"- **WHEN** {one_line(s.when)}", f"- **THEN** {one_line(s.then)}", ""]
|
|
95
|
+
return "\n".join(out)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _safe_id(capability_id: str) -> str:
|
|
99
|
+
return re.sub(r"[^a-z0-9_]+", "_", capability_id.lower()).strip("_") or "capability"
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def render_openspec(result: ExtractionResult, strict: bool | None = None, **options) -> RenderOutput:
|
|
103
|
+
"""A tree: specs/<capability>/spec.md per capability, NOT_EXPORTED.md and README.md at the
|
|
104
|
+
root. Strict (the default) leaves out a requirement with no scenario; backspec never writes one."""
|
|
105
|
+
strict = True if strict is None else strict
|
|
106
|
+
names = {c.id: c for c in result.capabilities}
|
|
107
|
+
exported: dict[str, list[Requirement]] = collections.defaultdict(list)
|
|
108
|
+
left_out: list[tuple[Requirement, str]] = []
|
|
109
|
+
for r in result.requirements:
|
|
110
|
+
reason = _not_exported_reason(r, strict)
|
|
111
|
+
if reason is None:
|
|
112
|
+
exported[r.capability].append(r)
|
|
113
|
+
else:
|
|
114
|
+
left_out.append((r, reason))
|
|
115
|
+
files: dict[str, str] = {}
|
|
116
|
+
paths: dict[str, str] = {}
|
|
117
|
+
for cid, requirements in exported.items():
|
|
118
|
+
capability = names.get(cid)
|
|
119
|
+
path = f"specs/{_safe_id(cid)}/spec.md"
|
|
120
|
+
paths[cid] = path
|
|
121
|
+
files[path] = _spec_file(capability.name if capability else cid, capability.summary if capability else "",
|
|
122
|
+
requirements)
|
|
123
|
+
|
|
124
|
+
def entry(r: Requirement, reason: str) -> str:
|
|
125
|
+
return f"- `{r.id}` ({r.capability}): {one_line(r.statement)}\n Reason: {reason}"
|
|
126
|
+
|
|
127
|
+
out = ["# Requirements not exported", ""]
|
|
128
|
+
if not left_out:
|
|
129
|
+
out += ["Every requirement was exported.", ""]
|
|
130
|
+
groups: dict[str, list[tuple[Requirement, str]]] = collections.defaultdict(list)
|
|
131
|
+
for r, reason in left_out:
|
|
132
|
+
groups[NO_SCENARIO if reason == NO_SCENARIO else "support below direct or partial"].append((r, reason))
|
|
133
|
+
for title, items in groups.items():
|
|
134
|
+
out += [f"## Not exported: {title}", ""] + [entry(r, reason) for r, reason in items] + [""]
|
|
135
|
+
files["NOT_EXPORTED.md"] = "\n".join(out)
|
|
136
|
+
|
|
137
|
+
reasons = collections.Counter(NO_SCENARIO if reason == NO_SCENARIO else reason.split(":")[0]
|
|
138
|
+
for _, reason in left_out)
|
|
139
|
+
count = sum(len(v) for v in exported.values())
|
|
140
|
+
readme = ["# OpenSpec export", "",
|
|
141
|
+
f"Written by backspec from a result with {len(result.requirements)} requirements "
|
|
142
|
+
f"({'strict' if strict else 'non-strict'} mode).", "",
|
|
143
|
+
f"- Exported: {count} requirements in {len(exported)} specs.",
|
|
144
|
+
f"- Not exported: {len(left_out)} requirements; see `NOT_EXPORTED.md`."]
|
|
145
|
+
readme += [f" - {n} with {reason}" for reason, n in sorted(reasons.items())]
|
|
146
|
+
if result.hidden:
|
|
147
|
+
readme.append(f"- {len(result.hidden)} requirements rejected in review are never exported.")
|
|
148
|
+
readme += ["", "Only requirements whose support is `direct` or `partial` are exported. backspec never writes a "
|
|
149
|
+
"scenario of its own: a scenario here comes from a test or example in the code, with its evidence.",
|
|
150
|
+
("In strict mode a requirement with no evidenced scenario is left out."
|
|
151
|
+
if strict else
|
|
152
|
+
"In non-strict mode a requirement with no scenario is included and marked as having none."), "",
|
|
153
|
+
"## Specs", ""]
|
|
154
|
+
for cid, path in paths.items():
|
|
155
|
+
capability = names.get(cid)
|
|
156
|
+
readme.append(f"- `{path}`: {capability.name if capability else cid}, {len(exported[cid])} requirements")
|
|
157
|
+
if not paths:
|
|
158
|
+
readme.append("- none")
|
|
159
|
+
readme.append("")
|
|
160
|
+
files["README.md"] = "\n".join(readme)
|
|
161
|
+
return RenderOutput(files=files)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
BODY_MARK = "<!--BACKSPEC_BODY-->"
|
|
165
|
+
PAGE = ('<!doctype html>\n<html lang="en">\n<head>\n<meta charset="utf-8">\n'
|
|
166
|
+
'<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">\n'
|
|
167
|
+
'{head}</head>\n<body>\n{body}</body>\n</html>\n')
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def page_data(result: ExtractionResult, title: str) -> dict:
|
|
171
|
+
"""What the page draws from: the result without the parts only a later run needs."""
|
|
172
|
+
data = result.model_dump(mode="json", exclude={"assignments": True, "observations": True,
|
|
173
|
+
"coverage": {"files": {"__all__": {"line_digest", "source_hash"}}}})
|
|
174
|
+
data["title"] = title
|
|
175
|
+
data["observations"] = [o.id for o in result.observations]
|
|
176
|
+
data["observations_by_id"] = {o.id: o.statement for o in result.observations}
|
|
177
|
+
return data
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def render_html(result: ExtractionResult, title: str | None = None, fragment: bool = False, **options) -> RenderOutput:
|
|
181
|
+
"""One self-contained page: the capabilities, each requirement with its evidence, what needs a
|
|
182
|
+
second look, and coverage.
|
|
183
|
+
It loads nothing from the network. `fragment` leaves out the document wrapper, for a host
|
|
184
|
+
that supplies its own."""
|
|
185
|
+
name = title or PurePath(result.run.root).name or "This codebase"
|
|
186
|
+
# "<" never appears raw in the embedded data, so nothing in a result can close the script element
|
|
187
|
+
data = json.dumps(page_data(result, name), ensure_ascii=False, separators=(",", ":")).replace("<", "\\u003c")
|
|
188
|
+
template = templates.load("report.html").replace("__BACKSPEC_TITLE__", _escape(f"{name} requirements"))
|
|
189
|
+
template = re.sub(r"__BACKSPEC_FONT:([a-z0-9.-]+)__",
|
|
190
|
+
lambda m: base64.b64encode(templates.load_bytes(m.group(1))).decode("ascii"), template)
|
|
191
|
+
template = template.replace("__BACKSPEC_DATA__", data)
|
|
192
|
+
if fragment:
|
|
193
|
+
return RenderOutput(text=template.replace(BODY_MARK + "\n", ""))
|
|
194
|
+
head, body = template.split(BODY_MARK + "\n", 1)
|
|
195
|
+
return RenderOutput(text=PAGE.format(head=head, body=body))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _escape(text: str) -> str:
|
|
199
|
+
return text.replace("&", "&").replace("<", "<").replace(">", ">")
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
RENDERERS: dict[str, Renderer] = {
|
|
203
|
+
"json": render_json,
|
|
204
|
+
"md": render_markdown,
|
|
205
|
+
"csv": render_csv,
|
|
206
|
+
"openspec": render_openspec,
|
|
207
|
+
"html": render_html,
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def formats() -> list[str]:
|
|
212
|
+
return list(RENDERERS)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def render(result: ExtractionResult, fmt: str, **options) -> RenderOutput:
|
|
216
|
+
"""The result in the named format. Unknown names are refused with the known ones."""
|
|
217
|
+
renderer = RENDERERS.get(fmt)
|
|
218
|
+
if renderer is None:
|
|
219
|
+
raise UnknownFormat(f"unknown format {fmt!r}; known formats: {', '.join(RENDERERS)}")
|
|
220
|
+
return renderer(result, **options)
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Assemble the coverage ledger and render a result for people to read."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import collections
|
|
5
|
+
|
|
6
|
+
from backspec.modules.discovery.facade import Discovery
|
|
7
|
+
from backspec.modules.evidence.facade import Earlier, ObservationSet, line_digest
|
|
8
|
+
from backspec.modules.report.domain import CoverageReport, FileCoverage, file_state, worth_looking_at
|
|
9
|
+
from backspec.modules.report.dto import Coverage, ExtractionResult, TriageAudit
|
|
10
|
+
from backspec.modules.requirements.facade import Requirement
|
|
11
|
+
from backspec.modules.triage.facade import EarlierVerdict, TriageResult, Verdict, clopper_pearson
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def build_coverage(discovery: Discovery, observed: ObservationSet, requirements: list[Requirement],
|
|
15
|
+
assignments: dict[str, str], triage: TriageResult | None = None) -> Coverage:
|
|
16
|
+
per_file = collections.Counter(o.path for o in observed.observations)
|
|
17
|
+
rejected = collections.Counter(r.path for r in observed.rejected)
|
|
18
|
+
nothing = {entry.path: entry.reason for entry in observed.nothing}
|
|
19
|
+
cited = collections.Counter(path for r in requirements for path in {e.path for e in r.evidence})
|
|
20
|
+
files = [FileCoverage(path=left.path, state=left.state, reason=left.reason) for left in discovery.left]
|
|
21
|
+
verdicts = triage.verdicts if triage else {}
|
|
22
|
+
audited = set(triage.audited) if triage else set()
|
|
23
|
+
for source in discovery.files:
|
|
24
|
+
verdict = verdicts.get(source.path)
|
|
25
|
+
out = bool(verdict and verdict.skipped) # triage skipped it
|
|
26
|
+
read = not out or source.path in audited
|
|
27
|
+
failed = observed.failed.get(source.path)
|
|
28
|
+
state, reason = file_state(per_file[source.path], rejected[source.path], failed, nothing.get(source.path))
|
|
29
|
+
if out and (not read or (state in ("no_requirements", "candidates_rejected"))):
|
|
30
|
+
# not read, or audited and yielded nothing: the triage role is the reason
|
|
31
|
+
state, reason = "triaged_out", verdict.role
|
|
32
|
+
files.append(FileCoverage(
|
|
33
|
+
path=source.path, state=state, reason=reason, observations=per_file[source.path],
|
|
34
|
+
rejected=rejected[source.path], lines=len(source.text.splitlines()),
|
|
35
|
+
analysis=observed.analysis.get(source.path, "text"), cited_by=cited[source.path], source_hash=source.source_hash,
|
|
36
|
+
line_digest=line_digest(source.lines) if read else "",
|
|
37
|
+
triage_role=verdict.role if verdict else "", triage_relevance=verdict.relevance if verdict else "",
|
|
38
|
+
triage_skipped=out, triage_audited=out and read,
|
|
39
|
+
found_by_audit=out and read and per_file[source.path] > 0))
|
|
40
|
+
files.sort(key=lambda f: f.path)
|
|
41
|
+
used = {e.observation for r in requirements for e in r.evidence}
|
|
42
|
+
unused = collections.Counter(assignments.get(o.id, "unassigned") for o in observed.observations if o.id not in used)
|
|
43
|
+
return Coverage(summary=dict(collections.Counter(f.state for f in files)), files=files,
|
|
44
|
+
unused_observations=dict(sorted(unused.items())),
|
|
45
|
+
triage_audit=_triage_audit(triage, per_file, observed.failed) if triage else None)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _triage_audit(triage: TriageResult, per_file: collections.Counter, failed: dict[str, str]) -> TriageAudit:
|
|
49
|
+
read = [path for path in triage.audited if path not in failed]
|
|
50
|
+
yielded = sorted(path for path in read if per_file[path] > 0)
|
|
51
|
+
return TriageAudit(skipped=len(triage.skipped), audited=len(read), yielded=len(yielded), yielded_paths=yielded,
|
|
52
|
+
failed_paths=sorted(path for path in triage.audited if path in failed),
|
|
53
|
+
miss_rate_interval=clopper_pearson(len(yielded), len(read)))
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def earlier_triage(previous: ExtractionResult, fingerprint: str) -> dict[str, EarlierVerdict]:
|
|
57
|
+
"""The verdicts a saved result reached, for a later run to keep for files that have not changed.
|
|
58
|
+
Empty when that result was triaged another way."""
|
|
59
|
+
if not fingerprint or previous.run.triage_fingerprint != fingerprint:
|
|
60
|
+
return {}
|
|
61
|
+
return {covered.path: EarlierVerdict(covered.source_hash, Verdict(covered.triage_role, covered.triage_relevance,
|
|
62
|
+
covered.triage_skipped))
|
|
63
|
+
for covered in previous.coverage.files if covered.source_hash and covered.triage_relevance}
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def coverage_report(result: ExtractionResult) -> CoverageReport:
|
|
67
|
+
"""What a saved result says about its files. No model and no file system: only the result."""
|
|
68
|
+
by_state: dict[str, list[FileCoverage]] = {}
|
|
69
|
+
for entry in result.coverage.files:
|
|
70
|
+
if entry.state != "excluded":
|
|
71
|
+
by_state.setdefault(entry.state, []).append(entry)
|
|
72
|
+
return CoverageReport(counts=dict(result.coverage.summary), files=dict(sorted(by_state.items())),
|
|
73
|
+
unused_observations=dict(result.coverage.unused_observations),
|
|
74
|
+
worth_a_look=[e for e in result.coverage.files if worth_looking_at(e)])
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def earlier_evidence(previous: ExtractionResult, fingerprint: str) -> dict[str, Earlier]:
|
|
78
|
+
"""What a saved result established about each file it read, for a later run to reuse.
|
|
79
|
+
Empty when that result observed files another way, or stopped on a failure for the file."""
|
|
80
|
+
if not fingerprint or previous.run.observe_fingerprint != fingerprint:
|
|
81
|
+
return {}
|
|
82
|
+
by_path: dict[str, list] = collections.defaultdict(list)
|
|
83
|
+
for observation in previous.observations:
|
|
84
|
+
by_path[observation.path].append(observation)
|
|
85
|
+
earlier = {}
|
|
86
|
+
for covered in previous.coverage.files:
|
|
87
|
+
audited_blank = covered.state == "triaged_out" and covered.triage_audited # read by the audit, nothing found
|
|
88
|
+
if covered.source_hash and (covered.state in ("extracted", "no_requirements") or audited_blank):
|
|
89
|
+
earlier[covered.path] = Earlier(
|
|
90
|
+
source_hash=covered.source_hash, observations=tuple(by_path.get(covered.path, [])),
|
|
91
|
+
nothing_reason=covered.reason if covered.state == "no_requirements" else
|
|
92
|
+
("audited; nothing verified" if audited_blank else None),
|
|
93
|
+
line_digest=covered.line_digest)
|
|
94
|
+
return earlier
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def to_markdown(result: ExtractionResult, include_technical: bool = False) -> str:
|
|
98
|
+
"""The catalogue grouped by capability, each requirement with its support and citations."""
|
|
99
|
+
by_capability: dict[str, list[Requirement]] = collections.defaultdict(list)
|
|
100
|
+
for requirement in result.requirements:
|
|
101
|
+
if include_technical or requirement.level != "technical":
|
|
102
|
+
by_capability[requirement.capability].append(requirement)
|
|
103
|
+
out = ["# Requirements", ""]
|
|
104
|
+
summary = ", ".join(f"{count} {state}" for state, count in sorted(result.coverage.summary.items()))
|
|
105
|
+
out += [f"{len(result.requirements)} requirements from {len(result.observations)} verified observations. "
|
|
106
|
+
f"Files: {summary}. Run status: {result.run.status}.", ""]
|
|
107
|
+
for capability in result.capabilities:
|
|
108
|
+
requirements = by_capability.get(capability.id)
|
|
109
|
+
if not requirements:
|
|
110
|
+
continue
|
|
111
|
+
out += [f"## {capability.name}", "", capability.summary, ""]
|
|
112
|
+
for requirement in requirements:
|
|
113
|
+
out.append(f"- **{requirement.statement}** `{requirement.id}` (support: {requirement.support})")
|
|
114
|
+
for condition in requirement.conditions:
|
|
115
|
+
out.append(f" - Condition: {condition}")
|
|
116
|
+
if requirement.support_note and requirement.support != "direct":
|
|
117
|
+
out.append(f" - Not shown: {requirement.support_note}")
|
|
118
|
+
cites = ", ".join(f"`{e.path}:{e.lines[0]}-{e.lines[1]}`" + (f" ({e.symbol})" if e.symbol else "") for e in requirement.evidence[:6])
|
|
119
|
+
more = f" and {len(requirement.evidence) - 6} more" if len(requirement.evidence) > 6 else ""
|
|
120
|
+
out.append(f" - Evidence: {cites}{more}")
|
|
121
|
+
for scenario in requirement.scenarios:
|
|
122
|
+
out += [f" - Scenario: {scenario.name}", f" - Given: {scenario.given}",
|
|
123
|
+
f" - When: {scenario.when}", f" - Then: {scenario.then}"]
|
|
124
|
+
for question in result.diagnostics.open_questions.get(capability.id, []):
|
|
125
|
+
out.append(f"- Open question: {question}")
|
|
126
|
+
out.append("")
|
|
127
|
+
return "\n".join(out)
|
|
File without changes
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
"""Requirements: complete behaviours, written from verified observations and then challenged.
|
|
2
|
+
|
|
3
|
+
Two rules hold by construction. A requirement cites observations by id and the program copies
|
|
4
|
+
their locations, so no citation in the output was written by a model at this step. And whether
|
|
5
|
+
the cited code supports the claim is a separate verdict from whether the citation exists.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import Literal, Protocol
|
|
10
|
+
|
|
11
|
+
from pydantic import BaseModel, Field
|
|
12
|
+
|
|
13
|
+
from backspec.kernel.text import numbered, short_hash
|
|
14
|
+
|
|
15
|
+
Support = Literal["direct", "partial", "insufficient", "conflicting", "not_assessed"]
|
|
16
|
+
CONTEXT_LINES = 10
|
|
17
|
+
MAX_EVIDENCE_SHOWN = 6
|
|
18
|
+
MAX_SPAN_SHOWN = 40
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Observed(Protocol):
|
|
22
|
+
"""What this module needs to know about a verified observation."""
|
|
23
|
+
id: str
|
|
24
|
+
kind: str
|
|
25
|
+
path: str
|
|
26
|
+
lines: tuple[int, int]
|
|
27
|
+
statement: str
|
|
28
|
+
quote: str
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class Evidence(BaseModel):
|
|
32
|
+
observation: str
|
|
33
|
+
role: Literal["establishes", "context"]
|
|
34
|
+
path: str
|
|
35
|
+
lines: tuple[int, int]
|
|
36
|
+
quote: str
|
|
37
|
+
symbol: str | None = None # copied from the observation
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class Scenario(BaseModel):
|
|
41
|
+
"""One concrete case, in the form given / when / then, resting on observations that show it."""
|
|
42
|
+
name: str
|
|
43
|
+
given: str
|
|
44
|
+
when: str
|
|
45
|
+
then: str
|
|
46
|
+
evidence: list[Evidence]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class Requirement(BaseModel):
|
|
50
|
+
id: str
|
|
51
|
+
capability: str
|
|
52
|
+
statement: str
|
|
53
|
+
conditions: list[str] = Field(default_factory=list)
|
|
54
|
+
level: Literal["user", "system", "technical"] = "user"
|
|
55
|
+
scope: Literal["local", "cross_file"]
|
|
56
|
+
support: Support = "not_assessed"
|
|
57
|
+
support_note: str = ""
|
|
58
|
+
review: Literal["unreviewed", "confirmed", "edited", "needs_review"] = "unreviewed"
|
|
59
|
+
evidence: list[Evidence]
|
|
60
|
+
merged_from: list[str] = Field(default_factory=list) # statements merged into this one
|
|
61
|
+
scenarios: list[Scenario] = Field(default_factory=list)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class CitedObservation(BaseModel):
|
|
65
|
+
observation: str
|
|
66
|
+
role: str = "establishes"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class RawScenario(BaseModel):
|
|
70
|
+
name: str | None = None
|
|
71
|
+
given: str | None = None
|
|
72
|
+
when: str | None = None
|
|
73
|
+
then: str | None = None
|
|
74
|
+
evidence: list[CitedObservation] = Field(default_factory=list)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class RawRequirement(BaseModel):
|
|
78
|
+
statement: str
|
|
79
|
+
conditions: list[str] = Field(default_factory=list)
|
|
80
|
+
level: str = "user"
|
|
81
|
+
evidence: list[CitedObservation] = Field(default_factory=list)
|
|
82
|
+
scenarios: list[RawScenario] = Field(default_factory=list)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class SynthReply(BaseModel):
|
|
86
|
+
requirements: list[RawRequirement] = Field(default_factory=list)
|
|
87
|
+
open_questions: list[str] = Field(default_factory=list)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class MergeGroup(BaseModel):
|
|
91
|
+
statement: str
|
|
92
|
+
conditions: list[str] = Field(default_factory=list)
|
|
93
|
+
level: str = "user"
|
|
94
|
+
merge: list[str] = Field(default_factory=list)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class ConsolidateReply(BaseModel):
|
|
98
|
+
groups: list[MergeGroup] = Field(default_factory=list)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class Verdict(BaseModel):
|
|
102
|
+
id: str
|
|
103
|
+
support: str
|
|
104
|
+
note: str = ""
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class AssessReply(BaseModel):
|
|
108
|
+
verdicts: list[Verdict] = Field(default_factory=list)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def synth_input(observations: list[Observed]) -> str:
|
|
112
|
+
out: list[str] = []
|
|
113
|
+
for o in sorted(observations, key=lambda o: (o.path, o.lines[0])):
|
|
114
|
+
out += [f"{o.id} | {o.kind} | {o.path}:{o.lines[0]}-{o.lines[1]}", f" {o.statement}",
|
|
115
|
+
f" quote: {o.quote[:160]}", ""]
|
|
116
|
+
return "\n".join(out)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _copied(cited: list[CitedObservation], known: dict[str, Observed]) -> list[Evidence]:
|
|
120
|
+
"""Evidence copied from the observations named; an id the model invented is dropped."""
|
|
121
|
+
evidence = []
|
|
122
|
+
for c in cited:
|
|
123
|
+
obs = known.get(c.observation)
|
|
124
|
+
if obs is not None:
|
|
125
|
+
evidence.append(Evidence(observation=obs.id, role="context" if c.role == "context" else "establishes",
|
|
126
|
+
path=obs.path, lines=obs.lines, quote=obs.quote,
|
|
127
|
+
symbol=getattr(obs, "symbol", None)))
|
|
128
|
+
return evidence
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def build_scenarios(raws: list[RawScenario], known: dict[str, Observed]) -> tuple[list[Scenario], int]:
|
|
132
|
+
"""The scenarios that are complete (name, given, when and then all non-empty) and cite at
|
|
133
|
+
least one real observation, with evidence copied from it; and how many were dropped."""
|
|
134
|
+
kept: list[Scenario] = []
|
|
135
|
+
dropped = 0
|
|
136
|
+
for raw in raws:
|
|
137
|
+
parts = [(p or "").strip() for p in (raw.name, raw.given, raw.when, raw.then)]
|
|
138
|
+
evidence = _copied(raw.evidence, known)
|
|
139
|
+
if not all(parts) or not evidence:
|
|
140
|
+
dropped += 1
|
|
141
|
+
continue
|
|
142
|
+
kept.append(Scenario(name=parts[0], given=parts[1], when=parts[2], then=parts[3], evidence=evidence))
|
|
143
|
+
return _distinct_scenarios(kept), dropped
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _scenario_key(s: Scenario) -> tuple[str, ...]:
|
|
147
|
+
return tuple(" ".join(t.lower().split()) for t in (s.name, s.given, s.when, s.then))
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _distinct_scenarios(scenarios: list[Scenario]) -> list[Scenario]:
|
|
151
|
+
"""The scenarios without repeats (same name and text), first one kept, order kept."""
|
|
152
|
+
seen: dict[tuple[str, ...], Scenario] = {}
|
|
153
|
+
for s in scenarios:
|
|
154
|
+
seen.setdefault(_scenario_key(s), s)
|
|
155
|
+
return list(seen.values())
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def build_requirement(raw: RawRequirement, capability: str, known: dict[str, Observed]) -> tuple[Requirement | None, int]:
|
|
159
|
+
"""A requirement with its evidence copied from the observations it names, or None when it
|
|
160
|
+
names no real observation; and the number of its scenarios that were dropped. Ids the model
|
|
161
|
+
invented are dropped, not trusted."""
|
|
162
|
+
evidence = _copied(raw.evidence, known)
|
|
163
|
+
if not evidence or not raw.statement.strip():
|
|
164
|
+
return None, 0
|
|
165
|
+
scenarios, dropped = build_scenarios(raw.scenarios, known)
|
|
166
|
+
level = raw.level if raw.level in ("user", "system", "technical") else "user"
|
|
167
|
+
return Requirement(
|
|
168
|
+
id="REQ-" + short_hash(capability, raw.statement), capability=capability,
|
|
169
|
+
statement=raw.statement.strip(), conditions=raw.conditions, level=level, # type: ignore[arg-type]
|
|
170
|
+
scope="cross_file" if len({e.path for e in evidence}) > 1 else "local", evidence=evidence,
|
|
171
|
+
scenarios=scenarios), dropped
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def build(raw: RawRequirement, capability: str, known: dict[str, Observed]) -> Requirement | None:
|
|
175
|
+
return build_requirement(raw, capability, known)[0]
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def consolidate_input(requirements: list[Requirement]) -> str:
|
|
179
|
+
out: list[str] = []
|
|
180
|
+
for r in requirements:
|
|
181
|
+
out += [f"{r.id} | {r.level}", f" statement: {r.statement}"]
|
|
182
|
+
if r.conditions:
|
|
183
|
+
out.append(" conditions: " + " ; ".join(r.conditions))
|
|
184
|
+
out += [" observations: " + ", ".join(e.observation for e in r.evidence), ""]
|
|
185
|
+
return "\n".join(out)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def merge_requirements(requirements: list[Requirement], groups: list[MergeGroup]) -> list[Requirement]:
|
|
189
|
+
"""The requirements with each model-proposed group merged into one. The program builds the
|
|
190
|
+
merged requirement: evidence is the union (by observation id, order kept) and scope is
|
|
191
|
+
recomputed. An unmentioned requirement is kept as it is. An unknown id is ignored, a
|
|
192
|
+
requirement named in two groups goes to the first, and a group left with fewer than two
|
|
193
|
+
known requirements merges nothing."""
|
|
194
|
+
by_id = {r.id: r for r in requirements}
|
|
195
|
+
claimed: set[str] = set()
|
|
196
|
+
merged_at: dict[str, Requirement] = {} # id of the first merged requirement -> the merged one
|
|
197
|
+
absorbed: set[str] = set()
|
|
198
|
+
for group in groups:
|
|
199
|
+
ids = [i for i in dict.fromkeys(group.merge) if i in by_id and i not in claimed]
|
|
200
|
+
if len(ids) < 2 or not group.statement.strip():
|
|
201
|
+
continue
|
|
202
|
+
claimed.update(ids)
|
|
203
|
+
sources = [by_id[i] for i in ids]
|
|
204
|
+
evidence: dict[str, Evidence] = {}
|
|
205
|
+
for source in sources:
|
|
206
|
+
for e in source.evidence:
|
|
207
|
+
evidence.setdefault(e.observation, e)
|
|
208
|
+
statement = group.statement.strip()
|
|
209
|
+
level = group.level if group.level in ("user", "system", "technical") else sources[0].level
|
|
210
|
+
capability = sources[0].capability
|
|
211
|
+
merged = Requirement(
|
|
212
|
+
id="REQ-" + short_hash(capability, statement), capability=capability, statement=statement,
|
|
213
|
+
conditions=group.conditions, level=level, # type: ignore[arg-type]
|
|
214
|
+
scope="cross_file" if len({e.path for e in evidence.values()}) > 1 else "local",
|
|
215
|
+
evidence=list(evidence.values()), merged_from=[s.statement for s in sources],
|
|
216
|
+
scenarios=_distinct_scenarios([sc for source in sources for sc in source.scenarios]))
|
|
217
|
+
merged_at[ids[0]] = merged
|
|
218
|
+
absorbed.update(ids[1:])
|
|
219
|
+
out: list[Requirement] = []
|
|
220
|
+
for r in requirements:
|
|
221
|
+
if r.id in absorbed:
|
|
222
|
+
continue
|
|
223
|
+
out.append(merged_at.get(r.id, r))
|
|
224
|
+
return _fold_same_id(out)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _fold_same_id(requirements: list[Requirement]) -> list[Requirement]:
|
|
228
|
+
"""Two requirements with one id (a merged statement equal to another's) become one, with the
|
|
229
|
+
evidence of both."""
|
|
230
|
+
out: dict[str, Requirement] = {}
|
|
231
|
+
for r in requirements:
|
|
232
|
+
first = out.get(r.id)
|
|
233
|
+
if first is None:
|
|
234
|
+
out[r.id] = r
|
|
235
|
+
continue
|
|
236
|
+
evidence = {e.observation: e for e in first.evidence}
|
|
237
|
+
for e in r.evidence:
|
|
238
|
+
evidence.setdefault(e.observation, e)
|
|
239
|
+
out[r.id] = first.model_copy(update={
|
|
240
|
+
"evidence": list(evidence.values()),
|
|
241
|
+
"scenarios": _distinct_scenarios([*first.scenarios, *r.scenarios]), "merged_from": first.merged_from + [r.statement],
|
|
242
|
+
"scope": "cross_file" if len({e.path for e in evidence.values()}) > 1 else "local"})
|
|
243
|
+
return list(out.values())
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def assess_block(requirement: Requirement, file_lines: dict[str, list[str]]) -> str:
|
|
247
|
+
"""One requirement with the code it cites: the cited lines marked `>`, with context."""
|
|
248
|
+
out = [f"##### {requirement.id}", f"STATEMENT: {requirement.statement}"]
|
|
249
|
+
if requirement.conditions:
|
|
250
|
+
out.append("CONDITIONS: " + " ; ".join(requirement.conditions))
|
|
251
|
+
for number, evidence in enumerate(requirement.evidence[:MAX_EVIDENCE_SHOWN], 1):
|
|
252
|
+
lines = file_lines[evidence.path]
|
|
253
|
+
first, last = evidence.lines[0], min(evidence.lines[1], evidence.lines[0] + MAX_SPAN_SHOWN)
|
|
254
|
+
start, end = max(1, first - CONTEXT_LINES), min(len(lines), last + CONTEXT_LINES)
|
|
255
|
+
out.append(f"--- evidence {number}: {evidence.path} lines {first}-{last}")
|
|
256
|
+
for text in numbered(lines[start - 1:end], start):
|
|
257
|
+
line_number = int(text[:5])
|
|
258
|
+
out.append((">" if first <= line_number <= last else " ") + text)
|
|
259
|
+
out.append("")
|
|
260
|
+
return "\n".join(out)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def settle(requirement: Requirement, verdict: Verdict | None) -> Requirement:
|
|
264
|
+
"""The requirement with its support verdict. Evidence that only shows something exists can
|
|
265
|
+
never earn more than `insufficient`, whatever the model said."""
|
|
266
|
+
if verdict is None or verdict.support not in ("direct", "partial", "insufficient", "conflicting"):
|
|
267
|
+
return requirement
|
|
268
|
+
support: Support = verdict.support # type: ignore[assignment]
|
|
269
|
+
note = verdict.note
|
|
270
|
+
if all(e.role == "context" for e in requirement.evidence) and support in ("direct", "partial"):
|
|
271
|
+
support, note = "insufficient", "Every cited observation only shows that something exists."
|
|
272
|
+
return requirement.model_copy(update={"support": support, "support_note": note})
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Requirements, as the rest of backspec sees it."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from backspec.kernel.llm import LLMPort
|
|
5
|
+
from backspec.modules.capabilities.facade import Capability
|
|
6
|
+
from backspec.modules.evidence.facade import Observation
|
|
7
|
+
from backspec.modules.requirements.domain import Evidence, Requirement, Scenario
|
|
8
|
+
from backspec.modules.requirements.use_cases import Synthesis, assess, fingerprint, synthesise
|
|
9
|
+
|
|
10
|
+
__all__ = ["Evidence", "Requirement", "RequirementsFacade", "Scenario", "Synthesis"]
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class RequirementsFacade:
|
|
14
|
+
def __init__(self, llm: LLMPort, workers: int = 4):
|
|
15
|
+
self._llm = llm
|
|
16
|
+
self._workers = workers
|
|
17
|
+
|
|
18
|
+
def synthesise(self, capabilities: list[Capability], by_capability: dict[str, list[Observation]]) -> Synthesis:
|
|
19
|
+
return synthesise(capabilities, by_capability, self._llm, self._workers)
|
|
20
|
+
|
|
21
|
+
def fingerprint(self) -> str:
|
|
22
|
+
"""Identifies the way requirements are written and judged; those made another way are not reused."""
|
|
23
|
+
return fingerprint()
|
|
24
|
+
|
|
25
|
+
def assess(self, requirements: list[Requirement], file_lines: dict[str, list[str]]) -> list[Requirement]:
|
|
26
|
+
return assess(requirements, file_lines, self._llm, self._workers)
|