agentforge-framework 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agentforge_framework/.claude-plugin/plugin.json +4 -0
- agentforge_framework/__init__.py +3 -0
- agentforge_framework/agents/__init__.py +92 -0
- agentforge_framework/agents/architect.py +146 -0
- agentforge_framework/agents/implementer.py +162 -0
- agentforge_framework/agents/orchestrator.py +588 -0
- agentforge_framework/agents/reviewer.py +335 -0
- agentforge_framework/agents/security.py +138 -0
- agentforge_framework/agents/tester.py +125 -0
- agentforge_framework/cli.py +461 -0
- agentforge_framework/context/__init__.py +1 -0
- agentforge_framework/context/extractors/__init__.py +76 -0
- agentforge_framework/context/extractors/base.py +47 -0
- agentforge_framework/context/extractors/python.py +65 -0
- agentforge_framework/context/extractors/sql.py +121 -0
- agentforge_framework/context/extractors/yaml.py +59 -0
- agentforge_framework/context/prompt.py +104 -0
- agentforge_framework/context/resolver.py +185 -0
- agentforge_framework/core/__init__.py +1 -0
- agentforge_framework/core/commands.py +170 -0
- agentforge_framework/core/config.py +90 -0
- agentforge_framework/core/contracts.py +875 -0
- agentforge_framework/core/gates.py +333 -0
- agentforge_framework/core/issues.py +697 -0
- agentforge_framework/core/plan_format.py +272 -0
- agentforge_framework/core/process.py +141 -0
- agentforge_framework/core/project.py +262 -0
- agentforge_framework/core/registry.py +455 -0
- agentforge_framework/core/repo.py +185 -0
- agentforge_framework/core/router.py +1 -0
- agentforge_framework/core/runtime.py +639 -0
- agentforge_framework/core/skills.py +255 -0
- agentforge_framework/core/workflow.py +215 -0
- agentforge_framework/plugins/__init__.py +35 -0
- agentforge_framework/plugins/databricks/__init__.py +86 -0
- agentforge_framework/plugins/pyspark/__init__.py +57 -0
- agentforge_framework/plugins/python/__init__.py +45 -0
- agentforge_framework/plugins/sql/__init__.py +377 -0
- agentforge_framework/providers/__init__.py +48 -0
- agentforge_framework/providers/base.py +248 -0
- agentforge_framework/providers/claude.py +159 -0
- agentforge_framework/providers/codex.py +139 -0
- agentforge_framework/skills/MANIFEST.yaml +157 -0
- agentforge_framework/skills/NOTICE +49 -0
- agentforge_framework/skills/domain-modeling/ADR-FORMAT.md +47 -0
- agentforge_framework/skills/domain-modeling/CONTEXT-FORMAT.md +60 -0
- agentforge_framework/skills/domain-modeling/SKILL.md +74 -0
- agentforge_framework/skills/domain-modeling/agents/openai.yaml +3 -0
- agentforge_framework/skills/grill-with-docs/SKILL.md +76 -0
- agentforge_framework/skills/grilling/SKILL.md +28 -0
- agentforge_framework/skills/grilling/agents/openai.yaml +3 -0
- agentforge_framework/skills/to-spec/SKILL.md +75 -0
- agentforge_framework/skills/to-spec/agents/openai.yaml +5 -0
- agentforge_framework/skills/to-tickets/SKILL.md +105 -0
- agentforge_framework/skills/to-tickets/agents/openai.yaml +5 -0
- agentforge_framework/skills/unslop/SKILL.md +131 -0
- agentforge_framework/skills/unslop/evals/fixtures/silhouette/human_reference.json +66 -0
- agentforge_framework/skills/unslop/scripts/_lang.py +106 -0
- agentforge_framework/skills/unslop/scripts/banned_phrase_scan.py +784 -0
- agentforge_framework/skills/unslop/scripts/calibrate_pairs.py +580 -0
- agentforge_framework/skills/unslop/scripts/calibrate_score.py +273 -0
- agentforge_framework/skills/unslop/scripts/check_packs.py +80 -0
- agentforge_framework/skills/unslop/scripts/check_suggestions.py +225 -0
- agentforge_framework/skills/unslop/scripts/contribute.py +373 -0
- agentforge_framework/skills/unslop/scripts/diff_check.py +139 -0
- agentforge_framework/skills/unslop/scripts/extract_constraints.py +201 -0
- agentforge_framework/skills/unslop/scripts/harvest_classify.py +223 -0
- agentforge_framework/skills/unslop/scripts/harvest_samples.py +534 -0
- agentforge_framework/skills/unslop/scripts/readability_metrics.py +295 -0
- agentforge_framework/skills/unslop/scripts/refresh_status.py +154 -0
- agentforge_framework/skills/unslop/scripts/silhouette_scan.py +390 -0
- agentforge_framework/skills/unslop/scripts/structure_scan.py +322 -0
- agentforge_framework/skills/unslop/scripts/suggest.py +211 -0
- agentforge_framework/skills/unslop/scripts/validate_preservation.py +409 -0
- agentforge_framework/skills/unslop/scripts/voice_card.py +496 -0
- agentforge_framework/skills/unslop/scripts/voice_profile.py +194 -0
- agentforge_framework/skills/unslop/scripts/voice_score.py +271 -0
- agentforge_framework/skills/unslop/scripts/wiki_sync.py +479 -0
- agentforge_framework/skills/write-plainly/SKILL.md +94 -0
- agentforge_framework/workflows/bugfix.yaml +8 -0
- agentforge_framework/workflows/feature.yaml +16 -0
- agentforge_framework/workflows/review.yaml +10 -0
- agentforge_framework-0.2.0.dist-info/METADATA +321 -0
- agentforge_framework-0.2.0.dist-info/RECORD +89 -0
- agentforge_framework-0.2.0.dist-info/WHEEL +5 -0
- agentforge_framework-0.2.0.dist-info/entry_points.txt +3 -0
- agentforge_framework-0.2.0.dist-info/licenses/LICENSE +202 -0
- agentforge_framework-0.2.0.dist-info/licenses/src/agentforge_framework/skills/NOTICE +49 -0
- agentforge_framework-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
"""Access to the skill bundle.
|
|
2
|
+
|
|
3
|
+
Skills ship as package data, never as importable modules. Markdown is read as
|
|
4
|
+
text; Python is invoked as a subprocess through the Command Runner, which is the
|
|
5
|
+
one process boundary in the codebase.
|
|
6
|
+
|
|
7
|
+
Most of the bundle is vendored third-party work (ADR-0006) and never edited in
|
|
8
|
+
place. A few skills are AgentForge's own; `FIRST_PARTY` names them, because a
|
|
9
|
+
refresh re-copies upstream over this directory and anything of ours that is not
|
|
10
|
+
findable goes with it.
|
|
11
|
+
|
|
12
|
+
One of ours is a composite: a skill whose job is to run two others together on
|
|
13
|
+
one task. `COMPOSED` says what each expands to, because a composite has to
|
|
14
|
+
survive both Capability Tiers — natively it fans out through the Skill tool, and
|
|
15
|
+
as a Fragment there is no tool to fan out with, so the delivery path inlines what
|
|
16
|
+
it names.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
import sys
|
|
23
|
+
from collections.abc import Sequence
|
|
24
|
+
from dataclasses import dataclass, field
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
|
|
27
|
+
from .process import CommandRunner, SubprocessRunner
|
|
28
|
+
|
|
29
|
+
SKILLS_ROOT = Path(__file__).resolve().parent.parent / "skills"
|
|
30
|
+
|
|
31
|
+
#: Scanners the `unslop` Command runs, in report order. The remaining scripts in
|
|
32
|
+
#: the bundle serve upstream's voice and calibration features and are not wired
|
|
33
|
+
#: into AgentForge.
|
|
34
|
+
UNSLOP_SCANNERS = (
|
|
35
|
+
"banned_phrase_scan.py",
|
|
36
|
+
"structure_scan.py",
|
|
37
|
+
"silhouette_scan.py",
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
#: Every skill in the bundle that AgentForge wrote. Not all of them are
|
|
42
|
+
#: composites — `write-plainly` is derived from what the scanners above enforce
|
|
43
|
+
#: and composes nothing — so this is the list a refresh has to be careful of,
|
|
44
|
+
#: and `COMPOSED` is not.
|
|
45
|
+
FIRST_PARTY: tuple[str, ...] = ("grill-with-docs", "write-plainly")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
#: The composites, and the vendored skills each is built out of. A composite adds
|
|
49
|
+
#: the job the parts are doing together and restates neither: a Fragment is the
|
|
50
|
+
#: degraded delivery of a skill and never a second copy of one, so the method
|
|
51
|
+
#: stays in exactly one file.
|
|
52
|
+
COMPOSED: dict[str, tuple[str, ...]] = {
|
|
53
|
+
"grill-with-docs": ("grilling", "domain-modeling"),
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def expand(names: Sequence[str]) -> tuple[str, ...]:
|
|
58
|
+
"""Declared skills, with each composite followed by what it is made of.
|
|
59
|
+
|
|
60
|
+
Order matters and duplicates do not survive it: the composite states the job
|
|
61
|
+
before the methods it draws on, and a Role that declared a part directly as
|
|
62
|
+
well gets it once.
|
|
63
|
+
"""
|
|
64
|
+
ordered: list[str] = []
|
|
65
|
+
for name in names:
|
|
66
|
+
for part in (name, *COMPOSED.get(name, ())):
|
|
67
|
+
if part not in ordered:
|
|
68
|
+
ordered.append(part)
|
|
69
|
+
return tuple(ordered)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class SkillNotFound(LookupError):
|
|
73
|
+
"""A skill was requested that is not in the vendored bundle."""
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def skill_path(name: str) -> Path:
|
|
77
|
+
"""Return the directory of a vendored skill."""
|
|
78
|
+
path = SKILLS_ROOT / name
|
|
79
|
+
if not path.is_dir():
|
|
80
|
+
available = ", ".join(sorted(p.name for p in SKILLS_ROOT.iterdir() if p.is_dir()))
|
|
81
|
+
raise SkillNotFound(f"no vendored skill named {name!r}; available: {available}")
|
|
82
|
+
return path
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def read_skill(name: str) -> str:
|
|
86
|
+
"""Return a skill's SKILL.md as text, for prompt-fragment delivery."""
|
|
87
|
+
return (skill_path(name) / "SKILL.md").read_text(encoding="utf-8")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@dataclass(frozen=True)
|
|
91
|
+
class ScanResult:
|
|
92
|
+
"""One scanner's verdict on one file."""
|
|
93
|
+
|
|
94
|
+
scanner: str
|
|
95
|
+
violations: int
|
|
96
|
+
clean: bool
|
|
97
|
+
report: dict | None = None
|
|
98
|
+
error: str | None = None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@dataclass(frozen=True)
|
|
102
|
+
class UnslopReport:
|
|
103
|
+
"""The aggregate verdict across every scanner."""
|
|
104
|
+
|
|
105
|
+
path: Path
|
|
106
|
+
results: list[ScanResult] = field(default_factory=list)
|
|
107
|
+
|
|
108
|
+
@property
|
|
109
|
+
def violations(self) -> int:
|
|
110
|
+
return sum(r.violations for r in self.results)
|
|
111
|
+
|
|
112
|
+
@property
|
|
113
|
+
def clean(self) -> bool:
|
|
114
|
+
return all(r.clean for r in self.results)
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def failed(self) -> list[ScanResult]:
|
|
118
|
+
"""Scanners that could not run at all, as distinct from ones that found faults."""
|
|
119
|
+
return [r for r in self.results if r.error is not None]
|
|
120
|
+
|
|
121
|
+
def to_dict(self) -> dict:
|
|
122
|
+
return {
|
|
123
|
+
"path": str(self.path),
|
|
124
|
+
"clean": self.clean,
|
|
125
|
+
"violations": self.violations,
|
|
126
|
+
"scanners": {
|
|
127
|
+
r.scanner: (
|
|
128
|
+
{"error": r.error}
|
|
129
|
+
if r.error
|
|
130
|
+
else {"violations": r.violations, "clean": r.clean, "report": r.report}
|
|
131
|
+
)
|
|
132
|
+
for r in self.results
|
|
133
|
+
},
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _count_violations(report: dict) -> int:
|
|
138
|
+
"""Pull a violation count out of a scanner report.
|
|
139
|
+
|
|
140
|
+
The scanners agree on exit codes but not on payload shape:
|
|
141
|
+
|
|
142
|
+
- ``banned_phrase_scan`` reports ``total_violations`` plus a ``violations`` list.
|
|
143
|
+
- ``structure_scan`` and ``silhouette_scan`` report a ``flags`` list and a
|
|
144
|
+
``flagged`` dict keyed by metric name.
|
|
145
|
+
|
|
146
|
+
Exit code remains the authority on pass or fail; this is only for reporting
|
|
147
|
+
how much was found.
|
|
148
|
+
"""
|
|
149
|
+
total = report.get("total_violations")
|
|
150
|
+
if isinstance(total, int):
|
|
151
|
+
return total
|
|
152
|
+
|
|
153
|
+
for key in ("violations", "flags"):
|
|
154
|
+
value = report.get(key)
|
|
155
|
+
if isinstance(value, list):
|
|
156
|
+
return len(value)
|
|
157
|
+
|
|
158
|
+
flagged = report.get("flagged")
|
|
159
|
+
if isinstance(flagged, dict):
|
|
160
|
+
return sum(1 for hit in flagged.values() if hit)
|
|
161
|
+
|
|
162
|
+
return 0
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _describe(result: ScanResult) -> list[str]:
|
|
166
|
+
"""One line per finding, using whichever detail keys the scanner provides."""
|
|
167
|
+
if not result.report:
|
|
168
|
+
return []
|
|
169
|
+
|
|
170
|
+
# `or ""` rather than a `get` default throughout: the scanners write an
|
|
171
|
+
# explicit null where they have no suggestion, and a default only fires on a
|
|
172
|
+
# missing key. The line read "'in today's' - None" until this was found by
|
|
173
|
+
# running the real scanners over real prose.
|
|
174
|
+
lines: list[str] = []
|
|
175
|
+
for violation in result.report.get("violations", []):
|
|
176
|
+
where = violation.get("line_number") or "?"
|
|
177
|
+
phrase = violation.get("phrase") or "?"
|
|
178
|
+
suggestion = violation.get("suggestion") or ""
|
|
179
|
+
lines.append(f"line {where}: {phrase!r} - {suggestion}".rstrip(" -"))
|
|
180
|
+
for flag in result.report.get("flags", []):
|
|
181
|
+
metric = flag.get("metric") or "?"
|
|
182
|
+
detail = flag.get("detail") or ""
|
|
183
|
+
suggestion = flag.get("suggestion") or ""
|
|
184
|
+
lines.append(f"{metric}: {detail} {suggestion}".strip())
|
|
185
|
+
return lines
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def render_report(report: UnslopReport) -> list[str]:
|
|
189
|
+
"""One line per scanner, and one per finding beneath it.
|
|
190
|
+
|
|
191
|
+
Shared by the Reviewer's Run Log entry and its own rewrite prompt, which is
|
|
192
|
+
the point: what a human reads about the prose and what the Role is asked to
|
|
193
|
+
act on are the same text, so neither can quietly say more than the other.
|
|
194
|
+
"""
|
|
195
|
+
lines: list[str] = []
|
|
196
|
+
for result in report.results:
|
|
197
|
+
if result.error:
|
|
198
|
+
lines.append(f"- {result.scanner}: could not run — {result.error}")
|
|
199
|
+
continue
|
|
200
|
+
verdict = "clean" if result.clean else f"{result.violations} finding(s)"
|
|
201
|
+
lines.append(f"- {result.scanner}: {verdict}")
|
|
202
|
+
lines += [f" - {line}" for line in _describe(result)]
|
|
203
|
+
return lines
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def run_unslop(
|
|
207
|
+
path: str | Path,
|
|
208
|
+
scanners: tuple[str, ...] = UNSLOP_SCANNERS,
|
|
209
|
+
runner: CommandRunner | None = None,
|
|
210
|
+
) -> UnslopReport:
|
|
211
|
+
"""Scan a file for machine-writing tells. Deterministic: no model involved."""
|
|
212
|
+
target = Path(path).resolve()
|
|
213
|
+
if not target.is_file():
|
|
214
|
+
raise FileNotFoundError(target)
|
|
215
|
+
|
|
216
|
+
runner = runner or SubprocessRunner()
|
|
217
|
+
scripts = skill_path("unslop") / "scripts"
|
|
218
|
+
results: list[ScanResult] = []
|
|
219
|
+
|
|
220
|
+
for scanner in scanners:
|
|
221
|
+
script = scripts / scanner
|
|
222
|
+
if not script.is_file():
|
|
223
|
+
results.append(
|
|
224
|
+
ScanResult(scanner, violations=0, clean=True, error=f"missing script: {script}")
|
|
225
|
+
)
|
|
226
|
+
continue
|
|
227
|
+
|
|
228
|
+
completed = runner.run([sys.executable, str(script), str(target)])
|
|
229
|
+
|
|
230
|
+
try:
|
|
231
|
+
report = json.loads(completed.stdout)
|
|
232
|
+
except json.JSONDecodeError:
|
|
233
|
+
detail = (completed.stderr or completed.stdout or "").strip()
|
|
234
|
+
results.append(
|
|
235
|
+
ScanResult(
|
|
236
|
+
scanner,
|
|
237
|
+
violations=0,
|
|
238
|
+
clean=True,
|
|
239
|
+
error=f"unparsable output (exit {completed.returncode}): {detail[:400]}",
|
|
240
|
+
)
|
|
241
|
+
)
|
|
242
|
+
continue
|
|
243
|
+
|
|
244
|
+
# The scanners exit 1 when they find something, so a non-zero exit is a
|
|
245
|
+
# verdict rather than a crash. Trust the payload for the count.
|
|
246
|
+
results.append(
|
|
247
|
+
ScanResult(
|
|
248
|
+
scanner,
|
|
249
|
+
violations=_count_violations(report),
|
|
250
|
+
clean=completed.returncode == 0,
|
|
251
|
+
report=report,
|
|
252
|
+
)
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
return UnslopReport(path=target, results=results)
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""Workflow definitions: the Roster as data rather than as Python.
|
|
2
|
+
|
|
3
|
+
A Workflow is a YAML-declared sequence of Roles with Gates between them. The
|
|
4
|
+
runtime walks the steps; it does not know which Roles exist, which is what makes
|
|
5
|
+
adding a seventh Role a definition and a line of YAML rather than an edit to the
|
|
6
|
+
engine.
|
|
7
|
+
|
|
8
|
+
Everything here happens before a Provider is invoked. A definition naming a Role
|
|
9
|
+
that cannot run, a Gate kind that does not exist, or a tier that was never a
|
|
10
|
+
tier is refused at load time, so a typo in a Workflow costs nothing rather than
|
|
11
|
+
costing a deep-tier planning pass.
|
|
12
|
+
|
|
13
|
+
A Step declares four things. The Role is resolved and run, the Model Tier
|
|
14
|
+
override is applied for that invocation, and the Gate is looked up in
|
|
15
|
+
`core.gates` and evaluated once the Step is behind the Run. The skip condition is
|
|
16
|
+
parsed and carried, waiting for the conditional-step work.
|
|
17
|
+
|
|
18
|
+
Every entry point here takes the Gate table to validate against, defaulting to
|
|
19
|
+
the shipped one. A Run with active Plugins passes the wider table
|
|
20
|
+
`core.registry` assembles, so a definition naming a Plugin's Gate kind loads
|
|
21
|
+
where that Plugin is active and is refused where it is not — which is the
|
|
22
|
+
honest answer, since nothing would evaluate it there. Nothing in this module
|
|
23
|
+
knows what a Plugin is.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
from collections.abc import Mapping
|
|
29
|
+
from dataclasses import dataclass
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
|
|
32
|
+
import yaml
|
|
33
|
+
|
|
34
|
+
from ..agents import UnknownRole, resolve_role
|
|
35
|
+
from .contracts import ModelTier, outstanding
|
|
36
|
+
from .gates import GATES, GateCheck
|
|
37
|
+
|
|
38
|
+
#: Shipped definitions live beside the package, like the vendored skills.
|
|
39
|
+
WORKFLOWS_ROOT = Path(__file__).resolve().parent.parent / "workflows"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class WorkflowError(ValueError):
|
|
43
|
+
"""A Workflow definition cannot be used. The message names the file and the fault."""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True)
|
|
47
|
+
class Step:
|
|
48
|
+
"""One Role invocation, plus the three things that qualify it.
|
|
49
|
+
|
|
50
|
+
`tier` overrides the Role's ADR-0004 default for this step alone. `gate`
|
|
51
|
+
names the kind of Gate that must clear before the next step begins. `when` is
|
|
52
|
+
the condition under which the step is skipped, and is not acted on yet.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
role: str
|
|
56
|
+
tier: ModelTier | None = None
|
|
57
|
+
gate: str | None = None
|
|
58
|
+
when: str | None = None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass(frozen=True)
|
|
62
|
+
class Workflow:
|
|
63
|
+
"""An ordered sequence of Steps, named."""
|
|
64
|
+
|
|
65
|
+
name: str
|
|
66
|
+
steps: tuple[Step, ...] = ()
|
|
67
|
+
|
|
68
|
+
def remaining(self, done: tuple[str, ...]) -> tuple[Step, ...]:
|
|
69
|
+
"""Steps that have not yet completed, in definition order."""
|
|
70
|
+
return outstanding(self.steps, done, lambda step: step.role)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def available_workflows(
|
|
74
|
+
root: Path | None = None, gates: Mapping[str, GateCheck] | None = None
|
|
75
|
+
) -> tuple[Workflow, ...]:
|
|
76
|
+
"""Every definition that loads, by name.
|
|
77
|
+
|
|
78
|
+
The Orchestrator picks one and names it in the Issue, so it has to be told
|
|
79
|
+
what there is. Read off the directory rather than listed in a prompt, so a
|
|
80
|
+
project that drops a definition beside the shipped ones can have it chosen.
|
|
81
|
+
A definition that does not load is left out rather than raising: one bad
|
|
82
|
+
file in the directory should not stop planning against the others.
|
|
83
|
+
"""
|
|
84
|
+
directory = Path(root) if root is not None else WORKFLOWS_ROOT
|
|
85
|
+
workflows = []
|
|
86
|
+
for path in sorted(directory.glob("*.yaml")):
|
|
87
|
+
try:
|
|
88
|
+
workflows.append(
|
|
89
|
+
parse_workflow(path.read_text(encoding="utf-8"), name=path.stem, gates=gates)
|
|
90
|
+
)
|
|
91
|
+
except WorkflowError:
|
|
92
|
+
continue
|
|
93
|
+
return tuple(workflows)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def load_workflow(
|
|
97
|
+
name: str, root: Path | None = None, gates: Mapping[str, GateCheck] | None = None
|
|
98
|
+
) -> Workflow:
|
|
99
|
+
"""Read and validate one definition by name."""
|
|
100
|
+
directory = Path(root) if root is not None else WORKFLOWS_ROOT
|
|
101
|
+
path = directory / f"{name}.yaml"
|
|
102
|
+
if not path.is_file():
|
|
103
|
+
available = ", ".join(sorted(p.stem for p in directory.glob("*.yaml")))
|
|
104
|
+
raise WorkflowError(
|
|
105
|
+
f"no Workflow named {name!r} in {directory}; available: {available or 'none'}"
|
|
106
|
+
)
|
|
107
|
+
return parse_workflow(path.read_text(encoding="utf-8"), name=name, gates=gates)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def parse_workflow(
|
|
111
|
+
text: str, *, name: str, gates: Mapping[str, GateCheck] | None = None
|
|
112
|
+
) -> Workflow:
|
|
113
|
+
"""Validate a definition's text. Every rejection names `name` and the fault."""
|
|
114
|
+
try:
|
|
115
|
+
data = yaml.safe_load(text)
|
|
116
|
+
except yaml.YAMLError as exc:
|
|
117
|
+
raise WorkflowError(f"Workflow {name!r} is not valid YAML: {exc}") from exc
|
|
118
|
+
|
|
119
|
+
if not isinstance(data, dict):
|
|
120
|
+
raise WorkflowError(f"Workflow {name!r} must be a mapping, not {_kind(data)}")
|
|
121
|
+
|
|
122
|
+
if "steps" not in data:
|
|
123
|
+
raise WorkflowError(f"Workflow {name!r} declares no `steps`")
|
|
124
|
+
|
|
125
|
+
raw_steps = data["steps"] or []
|
|
126
|
+
if not isinstance(raw_steps, list):
|
|
127
|
+
raise WorkflowError(f"Workflow {name!r}: `steps` must be a list, not {_kind(raw_steps)}")
|
|
128
|
+
|
|
129
|
+
steps = tuple(
|
|
130
|
+
_parse_step(raw, index=index, workflow=name, gates=gates)
|
|
131
|
+
for index, raw in enumerate(raw_steps, start=1)
|
|
132
|
+
)
|
|
133
|
+
return Workflow(name=str(data.get("name") or name), steps=steps)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _parse_step(
|
|
137
|
+
raw: object, *, index: int, workflow: str, gates: Mapping[str, GateCheck] | None
|
|
138
|
+
) -> Step:
|
|
139
|
+
where = f"Workflow {workflow!r} step {index}"
|
|
140
|
+
|
|
141
|
+
if not isinstance(raw, dict):
|
|
142
|
+
raise WorkflowError(f"{where} must be a mapping with a `role`, not {_kind(raw)}")
|
|
143
|
+
|
|
144
|
+
role = raw.get("role")
|
|
145
|
+
if not role or not isinstance(role, str):
|
|
146
|
+
raise WorkflowError(f"{where} declares no `role`")
|
|
147
|
+
|
|
148
|
+
try:
|
|
149
|
+
resolve_role(role)
|
|
150
|
+
except UnknownRole as exc:
|
|
151
|
+
raise WorkflowError(f"{where} names {role!r}: {exc}") from exc
|
|
152
|
+
|
|
153
|
+
return Step(
|
|
154
|
+
role=role.strip().lower(),
|
|
155
|
+
tier=_parse_tier(raw.get("tier"), where=where),
|
|
156
|
+
gate=_parse_gate(raw.get("gate"), where=where, gates=gates),
|
|
157
|
+
when=_optional_str(raw.get("when")),
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _parse_tier(value: object, *, where: str) -> ModelTier | None:
|
|
162
|
+
if value is None:
|
|
163
|
+
return None
|
|
164
|
+
try:
|
|
165
|
+
return ModelTier(str(value).strip().lower())
|
|
166
|
+
except ValueError as exc:
|
|
167
|
+
tiers = ", ".join(tier.value for tier in ModelTier)
|
|
168
|
+
raise WorkflowError(f"{where} names tier {value!r}; tiers are: {tiers}") from exc
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _parse_gate(
|
|
172
|
+
value: object, *, where: str, gates: Mapping[str, GateCheck] | None
|
|
173
|
+
) -> str | None:
|
|
174
|
+
"""A Gate kind is valid when something in this Run is registered to evaluate it.
|
|
175
|
+
|
|
176
|
+
Validated against the table rather than a list kept here: two lists would
|
|
177
|
+
let a definition name a Gate nothing answers for, which is a check that
|
|
178
|
+
silently never runs. `None` means the shipped table, and a Run hands in the
|
|
179
|
+
one its active Plugins widened.
|
|
180
|
+
|
|
181
|
+
The rejection names the kinds that are available rather than the kinds that
|
|
182
|
+
exist somewhere, because the reader's next question is what to write
|
|
183
|
+
instead: a definition naming a Plugin's Gate in a repository that Plugin does
|
|
184
|
+
not answer for is refused, and the list says so by leaving it out.
|
|
185
|
+
"""
|
|
186
|
+
if value is None:
|
|
187
|
+
return None
|
|
188
|
+
table = GATES if gates is None else gates
|
|
189
|
+
kind = str(value).strip().lower()
|
|
190
|
+
if kind not in table:
|
|
191
|
+
kinds = ", ".join(sorted(table))
|
|
192
|
+
raise WorkflowError(f"{where} names Gate kind {value!r}; kinds are: {kinds}")
|
|
193
|
+
return kind
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _optional_str(value: object) -> str | None:
|
|
197
|
+
if value is None:
|
|
198
|
+
return None
|
|
199
|
+
text = str(value).strip()
|
|
200
|
+
return text or None
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _kind(value: object) -> str:
|
|
204
|
+
return type(value).__name__
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
__all__ = [
|
|
208
|
+
"WORKFLOWS_ROOT",
|
|
209
|
+
"Step",
|
|
210
|
+
"Workflow",
|
|
211
|
+
"WorkflowError",
|
|
212
|
+
"available_workflows",
|
|
213
|
+
"load_workflow",
|
|
214
|
+
"parse_workflow",
|
|
215
|
+
]
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""AgentForge plugin packages.
|
|
2
|
+
|
|
3
|
+
`BUILT_IN` is the registry: a tuple rather than a dict or a set, because
|
|
4
|
+
activation order is the order Fragments reach a prompt in, and a reader
|
|
5
|
+
comparing two Run Logs should not have to wonder whether the order meant
|
|
6
|
+
anything. Registration is an entry here and a module beside it.
|
|
7
|
+
|
|
8
|
+
Order is also precedence where two Plugins claim one file suffix: the first
|
|
9
|
+
registered wins, and `core.registry.extractors_for` is where that is stated and
|
|
10
|
+
applied. `python` before `sql` is not a judgement about which matters more —
|
|
11
|
+
they claim disjoint suffixes — but the tuple is the tie-breaker the day two of
|
|
12
|
+
them do not.
|
|
13
|
+
|
|
14
|
+
The general before the specific, which is the order a reader wants them in: a
|
|
15
|
+
PySpark job is Python and is held to both, and the Fragment about annotating a
|
|
16
|
+
public function reaches its prompt before the one about the DataFrame API. The
|
|
17
|
+
four are also four different worked examples of the interface — `python`
|
|
18
|
+
contributes Fragments and nothing else, `sql` Extractors and nothing else,
|
|
19
|
+
`pyspark` is detected by what a file imports, and `databricks` by a marker at
|
|
20
|
+
the root and speaks differently to different Roles — so the fifth Plugin is
|
|
21
|
+
written by reading the one nearest to it rather than by reading the framework.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
from ..core.contracts import Plugin
|
|
27
|
+
from .databricks import DATABRICKS
|
|
28
|
+
from .pyspark import PYSPARK
|
|
29
|
+
from .python import PYTHON
|
|
30
|
+
from .sql import SQL
|
|
31
|
+
|
|
32
|
+
#: Every Plugin AgentForge ships, in the order they contribute.
|
|
33
|
+
BUILT_IN: tuple[Plugin, ...] = (PYTHON, SQL, PYSPARK, DATABRICKS)
|
|
34
|
+
|
|
35
|
+
__all__ = ["BUILT_IN", "DATABRICKS", "PYSPARK", "PYTHON", "SQL"]
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""The Databricks Plugin: what a workspace's code is held to, and what to audit.
|
|
2
|
+
|
|
3
|
+
A worked example of two things the other Plugins do not show.
|
|
4
|
+
|
|
5
|
+
**Detection by root marker alone.** A Databricks repository is one wherever the
|
|
6
|
+
work lands: a bundle's `databricks.yml` says the code in this tree is deployed
|
|
7
|
+
to a workspace, and that is true of a Plan touching one SQL file and of a Plan
|
|
8
|
+
touching one notebook. Unlike `pyspark` there is no import to read — the runtime
|
|
9
|
+
binds `spark` and `dbutils` for you, so a notebook that uses the whole platform
|
|
10
|
+
can import nothing at all — and unlike `python` the fact is a property of the
|
|
11
|
+
repository rather than of the blast radius.
|
|
12
|
+
|
|
13
|
+
**A Fragment that differs by Role.** What the Security Role needs to know about
|
|
14
|
+
a workspace is not what the Implementer needs. The Implementer is writing a
|
|
15
|
+
MERGE and needs the idiom this shop writes; the Security Role is auditing the
|
|
16
|
+
same file and needs to know where the secrets are meant to come from and which
|
|
17
|
+
grant is too wide. One Fragment each, keyed to the Roles that want them, because
|
|
18
|
+
the registry hands one Fragment per Plugin to one Role — the Implementer reading
|
|
19
|
+
a paragraph about service principals is paying for advice it cannot act on.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from ...core.contracts import Fragment, Plugin
|
|
25
|
+
|
|
26
|
+
#: What the Roles that write and review the code are held to. Three-part naming
|
|
27
|
+
#: and MERGE are here because they are what a reviewer rejects on sight and what
|
|
28
|
+
#: an Implementer reinvents every Run.
|
|
29
|
+
_CONVENTIONS = """\
|
|
30
|
+
Follow these Databricks conventions unless the code you are editing plainly does otherwise:
|
|
31
|
+
|
|
32
|
+
- Name every table in full: `catalog.schema.table`. A bare or two-part name
|
|
33
|
+
resolves against whatever `USE` ran last, so it reads correctly in the
|
|
34
|
+
notebook that wrote it and nowhere else. The catalog is configuration — take
|
|
35
|
+
it from the job's parameters rather than hard-coding the production one.
|
|
36
|
+
- Upsert with `MERGE INTO`, not delete-then-insert. Two statements are two
|
|
37
|
+
chances to leave the table short, and the window between them is one a
|
|
38
|
+
reader will hit.
|
|
39
|
+
- Match a MERGE on the business key, and write `WHEN MATCHED THEN UPDATE SET`
|
|
40
|
+
column by column rather than `SET *`, so a new source column cannot silently
|
|
41
|
+
overwrite a target one.
|
|
42
|
+
- Deduplicate the source before a MERGE. Two source rows for one key fail the
|
|
43
|
+
whole statement at run time, which is how a MERGE that passed on a sample
|
|
44
|
+
breaks in production.
|
|
45
|
+
- Do not repair a subset of rows with `INSERT OVERWRITE` on a managed table.
|
|
46
|
+
Delete or merge what is wrong.
|
|
47
|
+
- Change a table's layout with table properties, `OPTIMIZE`, and `ZORDER`
|
|
48
|
+
rather than by rewriting it. A rewrite breaks every reader mid-flight.\
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
#: The same repository read as a target rather than as a codebase. Every line is
|
|
52
|
+
#: something the Security Role can find in a diff and name a location for, which
|
|
53
|
+
#: is what its Findings are made of.
|
|
54
|
+
_POSTURE = """\
|
|
55
|
+
Audit Databricks code against these, and report what you find rather than fixing it:
|
|
56
|
+
|
|
57
|
+
- Secrets are fetched at run time from a secret scope — `dbutils.secrets.get`.
|
|
58
|
+
A token, storage key, or JDBC password written into a notebook, a job
|
|
59
|
+
definition, or a checked-in config is a Finding whatever the repository's
|
|
60
|
+
visibility, and so is a secret printed, logged, or written to a table.
|
|
61
|
+
- A job runs as a service principal holding its own grants. A personal access
|
|
62
|
+
token in a job definition or a bundle outlives the person it belongs to and
|
|
63
|
+
carries every permission they have.
|
|
64
|
+
- Access is granted in Unity Catalog on the narrowest object that answers the
|
|
65
|
+
need. A `GRANT` widened to the schema or the catalog to unblock one query is
|
|
66
|
+
the Finding, and so is `ALL PRIVILEGES` where `SELECT` was the requirement.
|
|
67
|
+
- Notebook widgets and job parameters are user input. Concatenated into a SQL
|
|
68
|
+
string they are injection; they belong in bound parameters.
|
|
69
|
+
- Sensitive data does not belong on the DBFS root, which every workspace user
|
|
70
|
+
can read. A Unity Catalog volume or an external location is where it goes.\
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
DATABRICKS = Plugin(
|
|
74
|
+
name="databricks",
|
|
75
|
+
# A bundle's descriptor under either spelling, and the CLI's profile file
|
|
76
|
+
# for a repository that predates bundles. Not `.databricks/`: that directory
|
|
77
|
+
# is local state the tooling writes and the repository ignores, so it says
|
|
78
|
+
# something about one machine rather than about the code.
|
|
79
|
+
root_markers=("databricks.yml", "databricks.yaml", ".databrickscfg"),
|
|
80
|
+
fragments=(
|
|
81
|
+
Fragment(text=_CONVENTIONS, roles=("implementer", "tester", "reviewer")),
|
|
82
|
+
Fragment(text=_POSTURE, roles=("security",)),
|
|
83
|
+
),
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
__all__ = ["DATABRICKS"]
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""The PySpark Plugin: what a Spark job is held to, wherever it lives.
|
|
2
|
+
|
|
3
|
+
A worked example of detection by import. `.py` is the suffix of a Spark job and
|
|
4
|
+
of a Django view and of this file, and holding all three to the DataFrame API
|
|
5
|
+
would be the fastest way to make a Plugin something people switch off. So this
|
|
6
|
+
Plugin declares no suffix at all: it answers for `pyspark`, and `core.registry`
|
|
7
|
+
reads the Python files the frozen Plan names to find out whether any of them
|
|
8
|
+
imports it. A repository with Spark jobs in it and a Plan that touches none of
|
|
9
|
+
them hears nothing, which is the same bargain the `python` Plugin makes by
|
|
10
|
+
declaring no root markers.
|
|
11
|
+
|
|
12
|
+
One Fragment, keyed to the three Roles that produce or judge code. The Security
|
|
13
|
+
Role is absent for the reason it is absent from the `python` Plugin: `.rdd` is
|
|
14
|
+
not a vulnerability, and a style convention in a security prompt competes with
|
|
15
|
+
the audit rather than supporting it. What the Security Role does need to know
|
|
16
|
+
about a Spark platform is a workspace's business, and the `databricks` Plugin
|
|
17
|
+
is where that Fragment lives.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from ...core.contracts import Fragment, Plugin
|
|
23
|
+
|
|
24
|
+
#: Every line here changes what an Agent produces. Spark conventions an Agent
|
|
25
|
+
#: would have followed anyway — name your DataFrames well, do not swallow
|
|
26
|
+
#: exceptions — are the `python` Fragment's job or nobody's, and repeating them
|
|
27
|
+
#: is tokens spent on agreement.
|
|
28
|
+
_CONVENTIONS = """\
|
|
29
|
+
Follow these PySpark conventions unless the module you are editing plainly does otherwise:
|
|
30
|
+
|
|
31
|
+
- Write DataFrame and Column expressions, not RDDs. `.rdd`, `map`, and
|
|
32
|
+
`flatMap` leave the optimiser out of the work, so a job that reaches for them
|
|
33
|
+
pays for a planner it refuses to use. Dropping to RDD for something the
|
|
34
|
+
DataFrame API already does is a defect.
|
|
35
|
+
- Reach for `pyspark.sql.functions`, imported as `F`, before writing a UDF. A
|
|
36
|
+
Python UDF serialises every row across the JVM boundary; where one is needed,
|
|
37
|
+
say in a comment which built-in was missing.
|
|
38
|
+
- Read with a declared schema. `inferSchema` reads the source twice and makes
|
|
39
|
+
the schema whatever last week's file happened to contain.
|
|
40
|
+
- Select the columns you need and join on named keys. A `select("*")` after a
|
|
41
|
+
join carries duplicated names downstream, and the error names the column
|
|
42
|
+
rather than the line that made it.
|
|
43
|
+
- Do not `collect()` or `toPandas()` a frame you have not bounded. Aggregate,
|
|
44
|
+
filter, or `limit` first: the driver holds whatever comes back.
|
|
45
|
+
- Tests build a local session and assert on small frames; one that needs a
|
|
46
|
+
cluster is not a test this repository can run.\
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
PYSPARK = Plugin(
|
|
50
|
+
name="pyspark",
|
|
51
|
+
imports=("pyspark",),
|
|
52
|
+
fragments=(
|
|
53
|
+
Fragment(text=_CONVENTIONS, roles=("implementer", "tester", "reviewer")),
|
|
54
|
+
),
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
__all__ = ["PYSPARK"]
|