activating-expertise 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. activating_expertise-0.1.0.dev0.dist-info/METADATA +28 -0
  2. activating_expertise-0.1.0.dev0.dist-info/RECORD +61 -0
  3. activating_expertise-0.1.0.dev0.dist-info/WHEEL +5 -0
  4. activating_expertise-0.1.0.dev0.dist-info/entry_points.txt +2 -0
  5. activating_expertise-0.1.0.dev0.dist-info/licenses/LICENSE-DOCS +29 -0
  6. activating_expertise-0.1.0.dev0.dist-info/licenses/LICENSE-SOFTWARE +661 -0
  7. activating_expertise-0.1.0.dev0.dist-info/top_level.txt +1 -0
  8. ae/__init__.py +0 -0
  9. ae/analysis.py +193 -0
  10. ae/cli.py +496 -0
  11. ae/concept_server.py +209 -0
  12. ae/concept_viewer.py +640 -0
  13. ae/concepts.py +155 -0
  14. ae/core.py +1303 -0
  15. ae/dspy_xp.py +894 -0
  16. ae/graph.py +733 -0
  17. ae/guides/__init__.py +0 -0
  18. ae/guides/debugging.md +434 -0
  19. ae/guides/design_maturation.md +141 -0
  20. ae/guides/hygienator.md +334 -0
  21. ae/guides/landscape_review.md +175 -0
  22. ae/guides/model_evaluation.md +313 -0
  23. ae/guides/model_onboarding.md +386 -0
  24. ae/guides/operational_guide.md +869 -0
  25. ae/guides/process_improvement.md +296 -0
  26. ae/guides/quick_start.md +123 -0
  27. ae/guides/roadmap.md +443 -0
  28. ae/guides/synergistic_xp.md +416 -0
  29. ae/seed_concepts/__init__.py +1 -0
  30. ae/seed_concepts/ae-concept-context.jsonld +62 -0
  31. ae/seed_concepts/ai-keeping-up.jsonld +121 -0
  32. ae/seed_concepts/bc-activities.jsonld +97 -0
  33. ae/seed_concepts/bounded-rationality.jsonld +88 -0
  34. ae/seed_concepts/collaborative-agent.jsonld +91 -0
  35. ae/seed_concepts/coming-to-know.jsonld +123 -0
  36. ae/seed_concepts/complement.jsonld +360 -0
  37. ae/seed_concepts/comprehensibility-as-participation.jsonld +101 -0
  38. ae/seed_concepts/constellation-architecture.jsonld +136 -0
  39. ae/seed_concepts/decision-network.jsonld +120 -0
  40. ae/seed_concepts/design-maturation.jsonld +71 -0
  41. ae/seed_concepts/discourse-ontology.jsonld +126 -0
  42. ae/seed_concepts/divergence-regimes.jsonld +94 -0
  43. ae/seed_concepts/dominator-pattern.jsonld +103 -0
  44. ae/seed_concepts/emergent-paradigm-adoption.jsonld +107 -0
  45. ae/seed_concepts/epistemic-integrity.jsonld +102 -0
  46. ae/seed_concepts/human-universe.jsonld +154 -0
  47. ae/seed_concepts/kinship.jsonld +122 -0
  48. ae/seed_concepts/override-safety.jsonld +97 -0
  49. ae/seed_concepts/partnership.jsonld +136 -0
  50. ae/seed_concepts/persona-activation-as-ceremony.jsonld +130 -0
  51. ae/seed_concepts/process-as-medium.jsonld +102 -0
  52. ae/seed_concepts/process-selection.jsonld +71 -0
  53. ae/seed_concepts/tangent-as-signal.jsonld +85 -0
  54. ae/serve.py +192 -0
  55. ae/session_server.py +305 -0
  56. ae/sync.py +134 -0
  57. ae/teams/__init__.py +0 -0
  58. ae/teams/problem_solving.md +34 -0
  59. ae/teams/problem_solving_complete.md +735 -0
  60. ae/view.py +1357 -0
  61. ae/work.py +815 -0
ae/dspy_xp.py ADDED
@@ -0,0 +1,894 @@
1
+ """dspy_xp — DSPy-controlled Synergistic XP pipeline for local models.
2
+
3
+ Replaces the monolithic-prompt approach (which stalls gemma4:26b) with a
4
+ DSPy-orchestrated pipeline where each Synergistic XP phase is a separate,
5
+ focused LLM call. The orchestrator handles state, tool execution, and
6
+ human checkpoints; DSPy modules handle LLM inference only.
7
+
8
+ Usage:
9
+ python -m ae.dspy_xp work_order.md --run-dir runs/test1
10
+ python -m ae.dspy_xp work_order.md --model openai/gemma4:31b
11
+ python -m ae.dspy_xp work_order.md --resume runs/test1
12
+
13
+ SPDX-License-Identifier: AGPL-3.0-only
14
+ """
15
+
16
+ import argparse
17
+ import json
18
+ import subprocess
19
+ import sys
20
+ import time
21
+ from datetime import datetime
22
+ from pathlib import Path
23
+
24
+ import dspy
25
+
26
+ # ---------------------------------------------------------------------------
27
+ # Constants
28
+ # ---------------------------------------------------------------------------
29
+
30
+ DEFAULT_MODEL = "openai/gemma4:26b"
31
+ DEFAULT_API_BASE = "http://100.77.22.113:8080/v1"
32
+ DEFAULT_API_KEY = "not-needed"
33
+ MAX_DEBUG_CYCLES = 3
34
+ MAX_FIX_CYCLES = 3
35
+
36
+ NUMPY_PITFALL = (
37
+ "CRITICAL: When building JSON output from numpy computations, wrap scalar "
38
+ "values with float(), int(), bool() before inserting them into dicts that "
39
+ "will be serialized with json.dump. Numpy scalars (np.float64, np.bool_, "
40
+ "etc.) are not JSON-serializable and will raise TypeError at write time."
41
+ )
42
+
43
+ # Phases after which the pipeline pauses for human input (unless --auto).
44
+ # Estimation included because it locks reference values every downstream
45
+ # check is measured against — last chance to catch "estimated the wrong
46
+ # thing" before committing compute to the build phase.
47
+ HUMAN_PAUSE_AFTER = {"story_writing", "design", "estimation", "complement_check"}
48
+
49
+
50
+ # ---------------------------------------------------------------------------
51
+ # DSPy Signatures — one per LLM-calling phase
52
+ # ---------------------------------------------------------------------------
53
+
54
+ class ProcessConfig(dspy.Signature):
55
+ """Analyze a work order and produce configuration for a Synergistic XP
56
+ development pipeline. The pipeline has these phases: team assembly,
57
+ story writing, complement criteria, design, estimation, build, execute,
58
+ debug, complement check, fix, report. Your job is to configure each
59
+ phase for THIS specific problem.
60
+
61
+ Read the work order carefully. Identify: what domain is this? What
62
+ kind of computation or artifact will be built? What are the key
63
+ technical decisions the design phase must address? What quantities
64
+ should be estimated before building? What does the build phase produce
65
+ (filename, language)? What common pitfalls exist for this type of code?
66
+ What output file does the solver/tool produce?"""
67
+
68
+ work_order: str = dspy.InputField(desc="Full work order text")
69
+ domain: str = dspy.OutputField(desc="Brief domain identification, e.g. 'structural engineering with FEM analysis'")
70
+ design_topics: str = dspy.OutputField(desc="Comma-separated list of technical decisions the design phase must address, e.g. 'element type, mesh strategy, material selection'")
71
+ estimation_focus: str = dspy.OutputField(desc="What quantities to estimate before building — formulas, reference values, assumptions to check")
72
+ build_description: str = dspy.OutputField(desc="What the build phase produces: language, purpose, e.g. 'Python FEM solver that computes stress and deflection'")
73
+ build_filename: str = dspy.OutputField(desc="Primary output script filename, e.g. 'fem_solver.py' or 'analyzer.py'")
74
+ results_filename: str = dspy.OutputField(desc="JSON results file the script writes, e.g. 'fem_results.json' or 'results.json'")
75
+ domain_pitfalls: str = dspy.OutputField(desc="Common coding pitfalls for this domain, e.g. 'numpy scalars not JSON-serializable; wrap with float()/int()/bool()'")
76
+ domain_experts: str = dspy.OutputField(desc="What domain expert roles the team needs, e.g. 'acoustician, materials scientist, computational mechanics specialist'")
77
+
78
+
79
+ class TeamAssembly(dspy.Signature):
80
+ """Assemble an expert team for this work order. Include the structural
81
+ team (Drucker — management, Eisler — partnership, Sawyer — group
82
+ creativity, Diderot — knowledge stewardship, Dewey — ethical practice)
83
+ plus Fowler (process integrity), Cunningham (design simplicity),
84
+ Beck (testing discipline), and domain experts as specified. Each member
85
+ states their perspective on the work and any initial concerns."""
86
+
87
+ work_order: str = dspy.InputField(desc="Full work order text")
88
+ domain_experts: str = dspy.InputField(desc="What domain expert roles to include")
89
+ team_roster: str = dspy.OutputField(desc="Team member introductions with roles, expertise, and initial perspectives")
90
+ deliberation: str = dspy.OutputField(desc="Team discussion about approach, risks, and who leads which aspect")
91
+
92
+
93
+ class StoryWriting(dspy.Signature):
94
+ """Write user stories with acceptance criteria for this work order.
95
+ The team shapes stories collaboratively — practitioner need first,
96
+ then technical shaping. Each story has: title, description,
97
+ acceptance criteria (testable), and scope exclusions."""
98
+
99
+ work_order: str = dspy.InputField()
100
+ team_roster: str = dspy.InputField()
101
+ stories: str = dspy.OutputField(desc="Numbered stories with acceptance criteria and scope exclusions")
102
+ deliberation: str = dspy.OutputField(desc="Team discussion shaping the stories — disagreements and resolutions")
103
+
104
+
105
+ class ComplementCriteria(dspy.Signature):
106
+ """Write complement criteria — what BAD work looks like — so the
107
+ complement check knows what to look for. Two categories:
108
+ (a) Process complements from Synergistic XP principles (5 items).
109
+ (b) Phase-specific complements from the acceptance criteria (at least 5).
110
+ Each criterion classified as hard (work would be wrong if matched) or
111
+ soft (legitimate alternative — document the tradeoff)."""
112
+
113
+ work_order: str = dspy.InputField()
114
+ stories: str = dspy.InputField()
115
+ complement_criteria: str = dspy.OutputField(desc="Numbered criteria, each with hard/soft classification and what evidence to look for")
116
+ deliberation: str = dspy.OutputField()
117
+
118
+
119
+ class Design(dspy.Signature):
120
+ """Team discusses the technical approach for this work order.
121
+ Address each topic listed in design_topics. Beck initiates the
122
+ verification plan. Cunningham challenges unnecessary complexity.
123
+ Domain experts contribute judgment from their specialization."""
124
+
125
+ work_order: str = dspy.InputField()
126
+ team_roster: str = dspy.InputField()
127
+ stories: str = dspy.InputField()
128
+ design_topics: str = dspy.InputField(desc="Technical decisions to address in this design discussion")
129
+ design_plan: str = dspy.OutputField(desc="Technical design decisions with rationale, verification plan, and parameter values")
130
+ coupling_parameters: str = dspy.OutputField(desc="Key parameter values shared across phases (geometry, material props, loads)")
131
+ deliberation: str = dspy.OutputField(desc="Team design discussion — alternatives considered, disagreements, and resolutions")
132
+
133
+
134
+ class Estimation(dspy.Signature):
135
+ """Produce first-principles estimates that every downstream computation
136
+ will be checked against. Focus on the quantities described in
137
+ estimation_focus. Decide which formulas apply, what each estimate
138
+ assumes. Show full derivations with units. Separate what you
139
+ estimated from what you deliberately left to the build phase
140
+ (the horizon complement)."""
141
+
142
+ work_order: str = dspy.InputField()
143
+ design_plan: str = dspy.InputField()
144
+ coupling_parameters: str = dspy.InputField()
145
+ estimation_focus: str = dspy.InputField(desc="What quantities to estimate and what to check")
146
+ estimates_code: str = dspy.OutputField(desc="Complete Python script that computes estimates and writes estimates.json")
147
+ estimates_json: str = dspy.OutputField(desc="The JSON output: estimates array + not_estimated array")
148
+ deliberation: str = dspy.OutputField()
149
+
150
+
151
+ class Build(dspy.Signature):
152
+ """Write the software described in build_description based on the design
153
+ plan and estimates. The code must be a single executable file that
154
+ reads no external config — all parameters inline. It writes a JSON
155
+ results file with entries that can be compared to the estimates.
156
+
157
+ IMPORTANT: Read domain_pitfalls carefully and avoid those issues.
158
+
159
+ NO STUBS. Every function body must do real work. Do NOT write
160
+ placeholder implementations, MOCK assemblies, `pass`-bodied functions,
161
+ TODO comments, or trivial approximations labeled "for demonstration."
162
+ If a computation is hard, implement it correctly; if it is beyond
163
+ scope, say so in a comment and raise NotImplementedError — do not
164
+ silently substitute a dummy value that will be used downstream."""
165
+
166
+ design_plan: str = dspy.InputField()
167
+ estimates_json: str = dspy.InputField()
168
+ coupling_parameters: str = dspy.InputField()
169
+ build_description: str = dspy.InputField(desc="What to build — language, purpose, output format")
170
+ domain_pitfalls: str = dspy.InputField(desc="Common coding pitfalls to avoid in this domain")
171
+ code: str = dspy.OutputField(desc="Complete executable code — the whole file")
172
+ deliberation: str = dspy.OutputField()
173
+
174
+
175
+ class Debug(dspy.Signature):
176
+ """Fix a code error. You receive the code that failed and its stderr.
177
+ Identify the root cause from the traceback, make the minimum targeted
178
+ fix, and return the complete fixed code. Do NOT rewrite from scratch.
179
+
180
+ If the error is a numpy type in json.dump, wrap with float()/int()/bool().
181
+ If an edit produces no change, try a completely different approach."""
182
+
183
+ code: str = dspy.InputField(desc="The code that produced the error")
184
+ stderr: str = dspy.InputField(desc="Full stderr including traceback")
185
+ estimates_json: str = dspy.InputField(desc="Reference values for sanity checking")
186
+ fixed_code: str = dspy.OutputField(desc="Complete fixed Python code — the whole file, not a diff")
187
+ fix_explanation: str = dspy.OutputField(desc="What was wrong and what you changed")
188
+ deliberation: str = dspy.OutputField()
189
+
190
+
191
+ class ComplementCheck(dspy.Signature):
192
+ """Check all artifacts against the complement criteria. For each
193
+ criterion, report: Match (Yes/Partial/No), Evidence (quote or
194
+ reference), Severity (hard/soft). Use operational guide Section 7b
195
+ Steps 1 (error complement), 2 (internal coherence), 3 (tradeoff
196
+ complement), 6 (synthesis and classification)."""
197
+
198
+ complement_criteria: str = dspy.InputField()
199
+ artifacts: str = dspy.InputField(desc="All phase outputs concatenated for checking")
200
+ findings: str = dspy.OutputField(desc="Per-criterion: Match, Evidence, Severity")
201
+ hard_count: int = dspy.OutputField(desc="Number of hard (must-fix) findings")
202
+ soft_count: int = dspy.OutputField(desc="Number of soft (tradeoff) findings")
203
+
204
+
205
+ class FixHard(dspy.Signature):
206
+ """Fix hard complement findings. You receive the specific findings
207
+ and the artifacts they apply to. Make targeted fixes — do NOT start
208
+ over. Return the fixed artifact text."""
209
+
210
+ artifacts: str = dspy.InputField()
211
+ findings: str = dspy.InputField(desc="Only the hard findings to fix")
212
+ fixed_artifacts: str = dspy.OutputField(desc="Fixed artifact content")
213
+ fix_explanation: str = dspy.OutputField()
214
+ deliberation: str = dspy.OutputField()
215
+
216
+
217
+ class Report(dspy.Signature):
218
+ """Generate a session report capturing all team dialogue, design
219
+ decisions, verification results, and complement check outcomes.
220
+ Preserve persona voices from the deliberation records."""
221
+
222
+ work_order: str = dspy.InputField()
223
+ full_deliberation: str = dspy.InputField(desc="All deliberation records from all phases, concatenated")
224
+ estimates_json: str = dspy.InputField()
225
+ execution_output: str = dspy.InputField(desc="Solver stdout and results")
226
+ complement_record: str = dspy.InputField()
227
+ report: str = dspy.OutputField(desc="Complete session report in markdown")
228
+
229
+
230
+ # ---------------------------------------------------------------------------
231
+ # Phase configuration
232
+ # ---------------------------------------------------------------------------
233
+
234
+ PHASES = [
235
+ {"name": "process_config", "signature": ProcessConfig, "has_tools": False},
236
+ {"name": "team_assembly", "signature": TeamAssembly, "has_tools": False},
237
+ {"name": "story_writing", "signature": StoryWriting, "has_tools": False},
238
+ {"name": "complement_criteria", "signature": ComplementCriteria, "has_tools": False},
239
+ {"name": "design", "signature": Design, "has_tools": False},
240
+ {"name": "estimation", "signature": Estimation, "has_tools": True},
241
+ {"name": "build", "signature": Build, "has_tools": True},
242
+ {"name": "execute", "signature": None, "has_tools": True}, # Python only
243
+ {"name": "debug", "signature": Debug, "has_tools": True}, # Conditional
244
+ {"name": "complement_check", "signature": ComplementCheck, "has_tools": False},
245
+ {"name": "fix_hard", "signature": FixHard, "has_tools": False}, # Conditional
246
+ {"name": "report", "signature": Report, "has_tools": True},
247
+ ]
248
+
249
+
250
+ # ---------------------------------------------------------------------------
251
+ # Tool functions
252
+ # ---------------------------------------------------------------------------
253
+
254
+ def strip_code_fences(text: str) -> str:
255
+ """Strip markdown code fences (```python ... ```) from LLM output."""
256
+ if not text:
257
+ return text
258
+ lines = text.strip().splitlines()
259
+ if lines and lines[0].strip().startswith("```"):
260
+ lines = lines[1:]
261
+ if lines and lines[-1].strip() == "```":
262
+ lines = lines[:-1]
263
+ return "\n".join(lines)
264
+
265
+
266
+ def write_file(path: Path, content: str) -> Path:
267
+ """Write content to a file. Returns the path."""
268
+ path.parent.mkdir(parents=True, exist_ok=True)
269
+ path.write_text(content)
270
+ return path
271
+
272
+
273
+ # Markers that indicate a model cheated instead of implementing. Checked
274
+ # against generated code before the execute phase so stub/mock
275
+ # implementations surface as findings the complement check can see.
276
+ STUB_MARKERS = [
277
+ (r"^\s*pass\s*$", "bare pass (empty function body)"),
278
+ (r"\bMOCK\b", "explicit MOCK marker"),
279
+ (r"\bTODO\b", "TODO comment"),
280
+ (r"\bFIXME\b", "FIXME comment"),
281
+ (r"\bPLACEHOLDER\b", "PLACEHOLDER marker"),
282
+ (r"for demonstration", "'for demonstration' — signals dummy implementation"),
283
+ (r"simplified.*demonstration", "'simplified demonstration' — signals dummy"),
284
+ (r"dummy", "'dummy' variable/value"),
285
+ (r"NotImplementedError", "NotImplementedError raised (may be deliberate — verify)"),
286
+ ]
287
+
288
+
289
+ def check_code_integrity(code: str) -> list[str]:
290
+ """Scan generated code for stub markers. Returns a list of findings.
291
+
292
+ Empty list means no stubs detected. Called after Build phase to surface
293
+ mock/placeholder implementations the in-pipeline complement check may
294
+ miss. Finding a NotImplementedError is listed but not presumed bad —
295
+ sometimes a deliberate scope-out.
296
+ """
297
+ import re
298
+
299
+ findings = []
300
+ for lineno, line in enumerate(code.splitlines(), 1):
301
+ # Skip comment-only lines for the text markers? No — MOCK in a
302
+ # comment is still a signal the generator took a shortcut.
303
+ for pattern, description in STUB_MARKERS:
304
+ if re.search(pattern, line, re.IGNORECASE | re.MULTILINE):
305
+ findings.append(f"line {lineno}: {description} — {line.strip()[:80]}")
306
+ break # one finding per line
307
+ return findings
308
+
309
+
310
+ def run_code(script_path: Path, timeout: int = 300) -> tuple[str, str, int]:
311
+ """Run a Python script. Returns (stdout, stderr, returncode).
312
+
313
+ On TimeoutExpired, returns partial output captured so far with rc=-1
314
+ so the orchestrator marks the phase failed and debug can run. On
315
+ other subprocess failures (executable missing, permissions), returns
316
+ an error message in stderr and rc=-2.
317
+ """
318
+ script_path = script_path.resolve()
319
+ try:
320
+ result = subprocess.run(
321
+ [sys.executable, str(script_path)],
322
+ capture_output=True, text=True, timeout=timeout,
323
+ cwd=str(script_path.parent),
324
+ )
325
+ return result.stdout, result.stderr, result.returncode
326
+ except subprocess.TimeoutExpired as e:
327
+ stdout = e.stdout.decode() if isinstance(e.stdout, bytes) else (e.stdout or "")
328
+ stderr = e.stderr.decode() if isinstance(e.stderr, bytes) else (e.stderr or "")
329
+ return stdout, f"{stderr}\n[TimeoutExpired after {timeout}s]", -1
330
+ except (OSError, FileNotFoundError) as e:
331
+ return "", f"Failed to launch subprocess: {e}", -2
332
+
333
+
334
+ # ---------------------------------------------------------------------------
335
+ # State management
336
+ # ---------------------------------------------------------------------------
337
+
338
+ def init_state(work_order_path: str, default_model: str, api_base: str,
339
+ phase_models: dict | None = None,
340
+ upstream_sessions: list[str] | None = None) -> dict:
341
+ """Create initial pipeline state."""
342
+ return {
343
+ "version": 1,
344
+ "pipeline": "synergistic_xp",
345
+ "work_order_path": work_order_path,
346
+ "default_model": default_model,
347
+ "api_base": api_base,
348
+ "phase_models": phase_models or {},
349
+ "upstream_sessions": upstream_sessions or [],
350
+ "current_phase": 0,
351
+ "status": "running",
352
+ "started": datetime.now().isoformat(),
353
+ "human_injections": [],
354
+ "phases": {},
355
+ }
356
+
357
+
358
+ def save_state(state: dict, run_dir: Path):
359
+ """Save pipeline state to JSON."""
360
+ path = run_dir / "session_state.json"
361
+ path.write_text(json.dumps(state, indent=2))
362
+
363
+
364
+ def load_state(run_dir: Path) -> dict:
365
+ """Load pipeline state from JSON."""
366
+ path = run_dir / "session_state.json"
367
+ return json.loads(path.read_text())
368
+
369
+
370
+ def get_model_for_phase(state: dict, phase_name: str) -> str:
371
+ """Get the model to use for a given phase."""
372
+ return state["phase_models"].get(phase_name, state["default_model"])
373
+
374
+
375
+ def configure_lm(model: str, api_base: str, max_tokens: int = 8192):
376
+ """Configure DSPy's LM for a phase."""
377
+ lm = dspy.LM(
378
+ model,
379
+ api_base=api_base,
380
+ api_key=DEFAULT_API_KEY,
381
+ max_tokens=max_tokens,
382
+ temperature=0.7,
383
+ )
384
+ dspy.configure(lm=lm)
385
+
386
+
387
+ def get_human_injections(state: dict, after_phase: str) -> str:
388
+ """Get any human injections targeting after a specific phase."""
389
+ injections = [h for h in state["human_injections"]
390
+ if h.get("after_phase") == after_phase]
391
+ if not injections:
392
+ return ""
393
+ parts = [f"[Human contribution ({h['timestamp']})]: {h['content']}"
394
+ for h in injections]
395
+ return "\n\n".join(parts)
396
+
397
+
398
+ def load_upstream_outputs(state: dict) -> dict:
399
+ """Merge all upstream session phase outputs into a single dict.
400
+
401
+ Keys collide on output name across sessions; later sessions win. Callers
402
+ that need per-session values should read the upstream state files directly.
403
+ Returns {} if no upstream sessions are configured or all fail to load.
404
+ """
405
+ merged: dict = {}
406
+ for session_path in state.get("upstream_sessions", []):
407
+ try:
408
+ upstream = json.loads(Path(session_path).read_text())
409
+ except (FileNotFoundError, json.JSONDecodeError) as e:
410
+ print(f" WARNING: upstream session unreadable ({session_path}): {e}")
411
+ continue
412
+ for phase_data in upstream.get("phases", {}).values():
413
+ outputs = phase_data.get("outputs", {})
414
+ for k, v in outputs.items():
415
+ if isinstance(v, str) and v:
416
+ merged[k] = v
417
+ return merged
418
+
419
+
420
+ # ---------------------------------------------------------------------------
421
+ # Human checkpoint
422
+ # ---------------------------------------------------------------------------
423
+
424
+ def human_checkpoint(phase_name: str, state: dict, run_dir: Path,
425
+ auto: bool) -> str | None:
426
+ """Pause for human review. Returns injection text, 'abort', or None."""
427
+ if auto:
428
+ return None
429
+
430
+ save_state(state, run_dir)
431
+ last_outputs = state["phases"].get(phase_name, {}).get("outputs", {})
432
+ delib = last_outputs.get("deliberation", "")[:2000]
433
+
434
+ print(f"\n{'='*60}")
435
+ print(f"PAUSE after {phase_name}")
436
+ print(f"{'='*60}")
437
+ if delib:
438
+ print(f"\nDeliberation summary:\n{delib}")
439
+ print(f"\nOptions: [c]ontinue [i]nject contribution [a]bort")
440
+
441
+ while True:
442
+ choice = input(f"\n{phase_name}> ").strip().lower()
443
+ if choice in ("c", "continue", ""):
444
+ return None
445
+ elif choice in ("i", "inject"):
446
+ print("Type your contribution (end with empty line):")
447
+ lines = []
448
+ while True:
449
+ line = input()
450
+ if not line:
451
+ break
452
+ lines.append(line)
453
+ contribution = "\n".join(lines)
454
+ state["human_injections"].append({
455
+ "after_phase": phase_name,
456
+ "timestamp": datetime.now().isoformat(),
457
+ "content": contribution,
458
+ })
459
+ save_state(state, run_dir)
460
+ print(f" Saved injection ({len(contribution)} chars)")
461
+ return contribution
462
+ elif choice in ("a", "abort"):
463
+ return "abort"
464
+ else:
465
+ print(" Enter c/i/a")
466
+
467
+
468
+ # ---------------------------------------------------------------------------
469
+ # Phase execution helpers
470
+ # ---------------------------------------------------------------------------
471
+
472
+ def phase_key(index: int, name: str) -> str:
473
+ return f"{index+1}_{name}"
474
+
475
+
476
+ def run_phase(phase_cfg: dict, index: int, state: dict, run_dir: Path,
477
+ auto: bool) -> bool:
478
+ """Execute a single phase. Returns True to continue, False to stop."""
479
+ name = phase_cfg["name"]
480
+ key = phase_key(index, name)
481
+
482
+ # Skip completed phases (resume support)
483
+ if key in state["phases"] and state["phases"][key].get("status") == "completed":
484
+ print(f" Phase {index+1} ({name}): already completed, skipping")
485
+ return True
486
+
487
+ state["current_phase"] = index
488
+ model = get_model_for_phase(state, name)
489
+ print(f"\n--- Phase {index+1}: {name} (model: {model}) ---")
490
+
491
+ t0 = time.time()
492
+ outputs = {}
493
+ artifacts_dir = run_dir / "artifacts"
494
+ artifacts_dir.mkdir(exist_ok=True)
495
+
496
+ # Gather inputs from upstream sessions, prior phases, and human injections.
497
+ # Upstream first so this-session phases override on key collision.
498
+ injection = get_human_injections(state, PHASES[index - 1]["name"] if index > 0 else "")
499
+ prior = load_upstream_outputs(state)
500
+ for prev_key, prev_data in state["phases"].items():
501
+ if prev_data.get("status") == "completed":
502
+ prior.update(prev_data.get("outputs", {}))
503
+
504
+ work_order = Path(state["work_order_path"]).read_text()
505
+
506
+ # --- Phase-specific logic ---
507
+
508
+ if name == "process_config":
509
+ configure_lm(model, state["api_base"])
510
+ result = dspy.ChainOfThought(ProcessConfig)(work_order=work_order)
511
+ outputs = {
512
+ "domain": result.domain,
513
+ "design_topics": result.design_topics,
514
+ "estimation_focus": result.estimation_focus,
515
+ "build_description": result.build_description,
516
+ "build_filename": result.build_filename or "solver.py",
517
+ "results_filename": result.results_filename or "results.json",
518
+ "domain_pitfalls": result.domain_pitfalls or NUMPY_PITFALL,
519
+ "domain_experts": result.domain_experts,
520
+ }
521
+ print(f" Domain: {result.domain}")
522
+ print(f" Build: {result.build_filename} → {result.results_filename}")
523
+ print(f" Pitfalls: {(result.domain_pitfalls or '')[:100]}")
524
+
525
+ elif name == "team_assembly":
526
+ configure_lm(model, state["api_base"])
527
+ wo = injection + "\n\n" + work_order if injection else work_order
528
+ result = dspy.ChainOfThought(TeamAssembly)(
529
+ work_order=wo,
530
+ domain_experts=prior.get("domain_experts", ""),
531
+ )
532
+ outputs = {"team_roster": result.team_roster, "deliberation": result.deliberation}
533
+
534
+ elif name == "story_writing":
535
+ configure_lm(model, state["api_base"])
536
+ wo = injection + "\n\n" + work_order if injection else work_order
537
+ result = dspy.ChainOfThought(StoryWriting)(
538
+ work_order=wo,
539
+ team_roster=prior.get("team_roster", ""),
540
+ )
541
+ outputs = {"stories": result.stories, "deliberation": result.deliberation}
542
+
543
+ elif name == "complement_criteria":
544
+ configure_lm(model, state["api_base"])
545
+ result = dspy.ChainOfThought(ComplementCriteria)(
546
+ work_order=work_order,
547
+ stories=prior.get("stories", ""),
548
+ )
549
+ outputs = {"complement_criteria": result.complement_criteria, "deliberation": result.deliberation}
550
+
551
+ elif name == "design":
552
+ configure_lm(model, state["api_base"])
553
+ wo = injection + "\n\n" + work_order if injection else work_order
554
+ result = dspy.ChainOfThought(Design)(
555
+ work_order=wo,
556
+ team_roster=prior.get("team_roster", ""),
557
+ stories=prior.get("stories", ""),
558
+ design_topics=prior.get("design_topics", ""),
559
+ )
560
+ outputs = {
561
+ "design_plan": result.design_plan,
562
+ "coupling_parameters": result.coupling_parameters,
563
+ "deliberation": result.deliberation,
564
+ }
565
+
566
+ elif name == "estimation":
567
+ configure_lm(model, state["api_base"], max_tokens=16384)
568
+ result = dspy.ChainOfThought(Estimation)(
569
+ work_order=work_order,
570
+ design_plan=prior.get("design_plan", ""),
571
+ coupling_parameters=prior.get("coupling_parameters", ""),
572
+ estimation_focus=prior.get("estimation_focus", ""),
573
+ )
574
+ outputs = {
575
+ "estimates_code": result.estimates_code,
576
+ "estimates_json": result.estimates_json,
577
+ "deliberation": result.deliberation,
578
+ }
579
+ # Write and run the estimates script
580
+ script = artifacts_dir / "estimates.py"
581
+ if not result.estimates_code:
582
+ print(f" WARNING: Estimation produced no code (truncated)")
583
+ else:
584
+ write_file(script, strip_code_fences(result.estimates_code))
585
+ stdout, stderr, rc = run_code(script)
586
+ if rc == 0:
587
+ print(f" estimates.py ran successfully")
588
+ # Try to read estimates.json from artifacts dir
589
+ est_path = artifacts_dir / "estimates.json"
590
+ if est_path.exists():
591
+ outputs["estimates_json"] = est_path.read_text()
592
+ else:
593
+ print(f" estimates.py failed: {stderr[:500]}")
594
+
595
+ elif name == "build":
596
+ configure_lm(model, state["api_base"], max_tokens=32768)
597
+ design = prior.get("design_plan", "")
598
+ if injection:
599
+ design = injection + "\n\n" + design
600
+ build_filename = prior.get("build_filename", "solver.py")
601
+ domain_pitfalls = prior.get("domain_pitfalls", NUMPY_PITFALL)
602
+ result = dspy.ChainOfThought(Build)(
603
+ design_plan=design,
604
+ estimates_json=prior.get("estimates_json", ""),
605
+ coupling_parameters=prior.get("coupling_parameters", ""),
606
+ build_description=prior.get("build_description", ""),
607
+ domain_pitfalls=domain_pitfalls,
608
+ )
609
+ outputs = {"code": result.code, "deliberation": result.deliberation}
610
+ if result.code:
611
+ clean_code = strip_code_fences(result.code)
612
+ write_file(artifacts_dir / build_filename, clean_code)
613
+ # Integrity check: scan for stub/mock/placeholder patterns
614
+ stub_findings = check_code_integrity(clean_code)
615
+ outputs["stub_findings"] = stub_findings
616
+ if stub_findings:
617
+ print(f" WARNING: build code contains {len(stub_findings)} stub/mock markers:")
618
+ for f in stub_findings[:5]:
619
+ print(f" - {f}")
620
+ if len(stub_findings) > 5:
621
+ print(f" ... and {len(stub_findings) - 5} more")
622
+ else:
623
+ print(f" WARNING: Build produced no code (likely truncated)")
624
+ outputs["code"] = ""
625
+ outputs["stub_findings"] = []
626
+
627
+ elif name == "execute":
628
+ # No LLM call — orchestrator runs the code
629
+ build_filename = prior.get("build_filename", "solver.py")
630
+ results_filename = prior.get("results_filename", "results.json")
631
+ solver = artifacts_dir / build_filename
632
+ if not solver.exists():
633
+ print(f" ERROR: {build_filename} not found")
634
+ outputs = {"stdout": "", "stderr": f"{build_filename} not found", "success": False}
635
+ else:
636
+ stdout, stderr, rc = run_code(solver)
637
+ outputs = {"stdout": stdout, "stderr": stderr, "success": rc == 0}
638
+ if rc == 0:
639
+ print(f" {build_filename} ran successfully")
640
+ results_path = artifacts_dir / results_filename
641
+ if results_path.exists():
642
+ outputs["results_json"] = results_path.read_text()
643
+ else:
644
+ print(f" {build_filename} failed (rc={rc}): {stderr[:500]}")
645
+
646
+ elif name == "debug":
647
+ # Only runs if execute failed
648
+ prev_exec = state["phases"].get(phase_key(index - 1, "execute"), {})
649
+ if prev_exec.get("outputs", {}).get("success", True):
650
+ print(f" Execute succeeded — skipping debug")
651
+ outputs = {"skipped": True}
652
+ else:
653
+ configure_lm(model, state["api_base"], max_tokens=32768)
654
+ code = prior.get("code", "")
655
+ stderr = prev_exec.get("outputs", {}).get("stderr", "")
656
+
657
+ for cycle in range(MAX_DEBUG_CYCLES):
658
+ print(f" Debug cycle {cycle + 1}/{MAX_DEBUG_CYCLES}")
659
+ result = dspy.ChainOfThought(Debug)(
660
+ code=code,
661
+ stderr=stderr,
662
+ estimates_json=prior.get("estimates_json", ""),
663
+ )
664
+ code = result.fixed_code
665
+ if not code:
666
+ print(f" WARNING: Debug produced no code (truncated), retrying")
667
+ continue
668
+ code = strip_code_fences(code)
669
+ build_fn = prior.get("build_filename", "solver.py")
670
+ results_fn = prior.get("results_filename", "results.json")
671
+ write_file(artifacts_dir / build_fn, code)
672
+ stdout, stderr, rc = run_code(artifacts_dir / build_fn)
673
+ if rc == 0:
674
+ print(f" Fixed on cycle {cycle + 1}")
675
+ outputs = {
676
+ "code": code,
677
+ "fix_explanation": result.fix_explanation,
678
+ "deliberation": result.deliberation,
679
+ "stdout": stdout,
680
+ "success": True,
681
+ }
682
+ # Update prior with fixed code
683
+ prior["code"] = code
684
+ results_path = artifacts_dir / results_fn
685
+ if results_path.exists():
686
+ outputs["results_json"] = results_path.read_text()
687
+ break
688
+ else:
689
+ print(f" Debug failed after {MAX_DEBUG_CYCLES} cycles")
690
+ outputs = {
691
+ "code": code,
692
+ "stderr": stderr,
693
+ "success": False,
694
+ "deliberation": result.deliberation,
695
+ }
696
+
697
+ elif name == "complement_check":
698
+ configure_lm(model, state["api_base"])
699
+ # Gather all artifacts for checking
700
+ artifact_parts = []
701
+ for k, v in prior.items():
702
+ if k not in ("deliberation",) and isinstance(v, str) and len(v) > 20:
703
+ artifact_parts.append(f"### {k}\n{v}")
704
+ artifacts_text = "\n\n".join(artifact_parts)
705
+
706
+ result = dspy.ChainOfThought(ComplementCheck)(
707
+ complement_criteria=prior.get("complement_criteria", ""),
708
+ artifacts=artifacts_text,
709
+ )
710
+ outputs = {
711
+ "findings": result.findings,
712
+ "hard_count": result.hard_count,
713
+ "soft_count": result.soft_count,
714
+ }
715
+ print(f" Complement: {result.hard_count} hard, {result.soft_count} soft")
716
+
717
+ elif name == "fix_hard":
718
+ # Only runs if complement_check found hard matches
719
+ cc = state["phases"].get(phase_key(index - 1, "complement_check"), {})
720
+ hard = cc.get("outputs", {}).get("hard_count", 0)
721
+ if not hard or hard == 0:
722
+ print(f" No hard findings — skipping fix")
723
+ outputs = {"skipped": True}
724
+ else:
725
+ configure_lm(model, state["api_base"], max_tokens=16384)
726
+ artifact_parts = []
727
+ for k, v in prior.items():
728
+ if k not in ("deliberation",) and isinstance(v, str) and len(v) > 20:
729
+ artifact_parts.append(f"### {k}\n{v}")
730
+ artifacts_text = "\n\n".join(artifact_parts)
731
+
732
+ findings = cc.get("outputs", {}).get("findings", "")
733
+ result = dspy.ChainOfThought(FixHard)(
734
+ artifacts=artifacts_text,
735
+ findings=findings,
736
+ )
737
+ outputs = {
738
+ "fixed_artifacts": result.fixed_artifacts,
739
+ "fix_explanation": result.fix_explanation,
740
+ "deliberation": result.deliberation,
741
+ }
742
+
743
+ elif name == "report":
744
+ configure_lm(model, state["api_base"], max_tokens=16384)
745
+ # Collect all deliberations
746
+ all_delib = []
747
+ for pk, pv in sorted(state["phases"].items()):
748
+ d = pv.get("outputs", {}).get("deliberation", "")
749
+ if d:
750
+ all_delib.append(f"## {pk}\n{d}")
751
+ full_delib = "\n\n".join(all_delib)
752
+
753
+ exec_output = prior.get("stdout", "") + "\n" + prior.get("results_json", "")
754
+ complement_record = prior.get("findings", "")
755
+
756
+ result = dspy.ChainOfThought(Report)(
757
+ work_order=work_order,
758
+ full_deliberation=full_delib,
759
+ estimates_json=prior.get("estimates_json", ""),
760
+ execution_output=exec_output,
761
+ complement_record=complement_record,
762
+ )
763
+ outputs = {"report": result.report}
764
+ write_file(run_dir / "session_report.md", result.report)
765
+
766
+ elapsed = time.time() - t0
767
+ state["phases"][key] = {
768
+ "status": "completed",
769
+ "model_used": model,
770
+ "time_seconds": round(elapsed, 1),
771
+ "outputs": outputs,
772
+ }
773
+ save_state(state, run_dir)
774
+ print(f" Done ({elapsed:.1f}s)")
775
+
776
+ # Human checkpoint
777
+ if name in HUMAN_PAUSE_AFTER:
778
+ result = human_checkpoint(name, state, run_dir, auto)
779
+ if result == "abort":
780
+ state["status"] = "aborted"
781
+ save_state(state, run_dir)
782
+ return False
783
+
784
+ return True
785
+
786
+
787
+ # ---------------------------------------------------------------------------
788
+ # Pipeline orchestrator
789
+ # ---------------------------------------------------------------------------
790
+
791
+ def run_xp_pipeline(work_order_path: Path, run_dir: Path,
792
+ default_model: str = DEFAULT_MODEL,
793
+ api_base: str = DEFAULT_API_BASE,
794
+ phase_models: dict | None = None,
795
+ upstream_sessions: list[str] | None = None,
796
+ auto: bool = False,
797
+ resume: bool = False) -> bool:
798
+ """Run the full Synergistic XP pipeline. Returns True on success."""
799
+ run_dir.mkdir(parents=True, exist_ok=True)
800
+
801
+ if resume and (run_dir / "session_state.json").exists():
802
+ print(f"Resuming from {run_dir / 'session_state.json'}")
803
+ state = load_state(run_dir)
804
+ else:
805
+ state = init_state(
806
+ str(work_order_path.resolve()),
807
+ default_model, api_base,
808
+ phase_models, upstream_sessions,
809
+ )
810
+
811
+ print(f"Pipeline: synergistic_xp ({len(PHASES)} phases)")
812
+ print(f"Model: {default_model}")
813
+ print(f"API: {api_base}")
814
+ print(f"Run dir: {run_dir}")
815
+ print(f"Auto: {auto}")
816
+ if phase_models:
817
+ print(f"Phase models: {phase_models}")
818
+ if state.get("upstream_sessions"):
819
+ print(f"Upstream sessions: {len(state['upstream_sessions'])}")
820
+ for p in state["upstream_sessions"]:
821
+ print(f" - {p}")
822
+
823
+ for i, phase_cfg in enumerate(PHASES):
824
+ ok = run_phase(phase_cfg, i, state, run_dir, auto)
825
+ if not ok:
826
+ print(f"\nPipeline stopped at phase {i+1} ({phase_cfg['name']})")
827
+ return False
828
+
829
+ state["status"] = "completed"
830
+ state["finished"] = datetime.now().isoformat()
831
+ save_state(state, run_dir)
832
+
833
+ # Print summary
834
+ print(f"\n{'='*60}")
835
+ print(f"PIPELINE COMPLETE")
836
+ print(f"{'='*60}")
837
+ total = sum(p.get("time_seconds", 0) for p in state["phases"].values())
838
+ print(f"Total time: {total:.0f}s ({total/60:.1f} min)")
839
+ for key, data in sorted(state["phases"].items()):
840
+ status = data.get("status", "?")
841
+ t = data.get("time_seconds", 0)
842
+ model = data.get("model_used", "?")
843
+ skipped = data.get("outputs", {}).get("skipped", False)
844
+ suffix = " (skipped)" if skipped else ""
845
+ print(f" {key}: {status} ({t:.1f}s, {model}){suffix}")
846
+
847
+ return True
848
+
849
+
850
+ # ---------------------------------------------------------------------------
851
+ # CLI
852
+ # ---------------------------------------------------------------------------
853
+
854
+ def main():
855
+ parser = argparse.ArgumentParser(description="DSPy-controlled Synergistic XP pipeline")
856
+ parser.add_argument("work_order", type=Path, help="Path to work order markdown file")
857
+ parser.add_argument("--run-dir", type=Path, default=None,
858
+ help="Run directory (default: runs/<timestamp>)")
859
+ parser.add_argument("--model", default=DEFAULT_MODEL,
860
+ help=f"Default model (default: {DEFAULT_MODEL})")
861
+ parser.add_argument("--api-base", default=DEFAULT_API_BASE,
862
+ help=f"API base URL (default: {DEFAULT_API_BASE})")
863
+ parser.add_argument("--phase-model", nargs=2, action="append", default=[],
864
+ metavar=("PHASE", "MODEL"),
865
+ help="Override model for a phase (e.g., --phase-model build openai/gemma4:31b)")
866
+ parser.add_argument("--upstream", type=str, action="append", default=[],
867
+ help="Path to upstream session_state.json")
868
+ parser.add_argument("--auto", action="store_true",
869
+ help="Skip human checkpoints")
870
+ parser.add_argument("--resume", action="store_true",
871
+ help="Resume from existing session_state.json in run-dir")
872
+
873
+ args = parser.parse_args()
874
+
875
+ if args.run_dir is None:
876
+ args.run_dir = Path("runs") / datetime.now().strftime("%Y%m%d_%H%M%S")
877
+
878
+ phase_models = {p: m for p, m in args.phase_model}
879
+
880
+ success = run_xp_pipeline(
881
+ work_order_path=args.work_order,
882
+ run_dir=args.run_dir,
883
+ default_model=args.model,
884
+ api_base=args.api_base,
885
+ phase_models=phase_models,
886
+ upstream_sessions=args.upstream,
887
+ auto=args.auto,
888
+ resume=args.resume,
889
+ )
890
+ sys.exit(0 if success else 1)
891
+
892
+
893
+ if __name__ == "__main__":
894
+ main()