pi-dynamic-workflows-py 0.1.0__py3-none-any.whl

pi_dynamic_workflows/__init__.py

@@ -0,0 +1,133 @@
+"""pi-dynamic-workflows — Dynamic workflow orchestration for pi-python.
+
+Provides:
+  - ``workflow`` tool for multi-agent orchestration via Python scripts
+  - ``/workflows`` command for listing and managing runs
+  - 5 built-in workflow patterns: deep-research, adversarial-review,
+    code-review, multi-perspective, codebase-audit
+  - Model tier routing (small/medium/big)
+  - Token budget tracking
+
+Install: ``pip install pi-dynamic-workflows-py``
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+from pi_dynamic_workflows.builtin_workflows import (
+    BUILTIN_WORKFLOW_NAMES,
+    BUILTIN_WORKFLOWS,
+)
+from pi_dynamic_workflows.builtin_workflows import (
+    resolve_builtin_workflow as resolve_builtin_workflow,
+)
+from pi_dynamic_workflows.workflow_tool import create_workflow_tool
+
+if TYPE_CHECKING:
+    from pi_agent_core.extensions import ExtensionAPI
+
+logger = logging.getLogger(__name__)
+
+
+def _handle_workflows_command(pi: ExtensionAPI, args: str) -> None:
+    """Handler for ``/workflows`` command."""
+    parts = args.strip().split(maxsplit=1)
+    sub = parts[0] if parts else "list"
+
+    if sub == "list" or not sub:
+        names = ", ".join(BUILTIN_WORKFLOW_NAMES)
+        pi.send_message(f"Available built-in workflows: {names}")
+    elif sub == "help":
+        lines = ["## Built-in Workflow Patterns\n"]
+        for name, desc in BUILTIN_WORKFLOWS.items():
+            lines.append(f"- **{name}**: {desc.description}")
+        lines.append("\nUse the `workflow` tool with `name` parameter to run a built-in pattern.")
+        pi.send_message("\n".join(lines))
+    else:
+        pi.send_message(f"Unknown subcommand: {sub}\nUsage: /workflows [list|help]")
+
+
+def _register_builtin_commands(pi: ExtensionAPI) -> None:
+    """Register slash commands for each built-in workflow pattern.
+
+    These are passthrough commands: advertised for client autocomplete
+    but forwarded to the LLM so it invokes the ``workflow`` tool.
+    """
+    _noop = lambda args: None  # noqa: E731
+    for name, desc in BUILTIN_WORKFLOWS.items():
+        pi.register_command(
+            name,
+            description=desc.description,
+            handler=_noop,
+            passthrough=True,
+        )
+
+
+def _try_build_real_executor(pi: ExtensionAPI) -> tuple:
+    """Try to build a HarnessSubagentExecutor from the bridge. Returns (executor, manager)."""
+    try:
+        bridge = pi._require_bridge()
+        stream_fn = getattr(bridge, "stream_fn", None)
+        model = getattr(bridge, "model", None)
+        get_api_key = getattr(bridge, "get_api_key_fn", None)
+        if stream_fn is None or model is None:
+            return None, None
+
+        from pi_dynamic_workflows.manager import WorkflowManager
+        from pi_dynamic_workflows.subagent import HarnessSubagentExecutor
+
+        executor = HarnessSubagentExecutor(
+            stream_fn=stream_fn,
+            parent_model=model,
+            cwd=pi.cwd,
+            get_api_key=get_api_key,
+        )
+        manager = WorkflowManager(bridge)
+        return executor, manager
+    except Exception:
+        logger.debug("Could not build real executor; falling back to mock", exc_info=True)
+        return None, None
+
+
+def _register_saved_workflows(pi: ExtensionAPI) -> None:
+    """Scan saved workflow directories and register slash commands."""
+    try:
+        from pi_dynamic_workflows.store import WorkflowStore
+
+        store = WorkflowStore(cwd=pi.cwd)
+        for wf in store.scan():
+            if wf.name in BUILTIN_WORKFLOW_NAMES:
+                continue
+            pi.register_command(
+                wf.name,
+                description=wf.description or f"Run saved workflow: {wf.name}",
+                handler=lambda args, _wf=wf: pi.send_message(
+                    f"To run the saved workflow **{_wf.name}**, use the `workflow` tool with:\n"
+                    f"  script=<contents of {_wf.path}>\n\n"
+                    f"Description: {_wf.description}"
+                ),
+            )
+    except Exception:
+        logger.debug("Saved workflow scan failed", exc_info=True)
+
+
+def activate(pi: ExtensionAPI) -> None:
+    """Extension entry point — called by the ExtensionLoader."""
+    executor, manager = _try_build_real_executor(pi)
+
+    pi.register_tool(create_workflow_tool(executor=executor, cwd=pi.cwd, manager=manager))
+
+    pi.register_command(
+        "workflows",
+        description="List and manage dynamic workflows",
+        handler=lambda args: _handle_workflows_command(pi, args if isinstance(args, str) else ""),
+    )
+
+    _register_builtin_commands(pi)
+    _register_saved_workflows(pi)
+
+    if manager is not None:
+        bridge = pi._require_bridge()
+        bridge.register_cleanup(manager.shutdown)

pi_dynamic_workflows/budget.py

@@ -0,0 +1,42 @@
+"""Token budget tracking for workflow runs."""
+
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass, field
+
+
+@dataclass
+class TokenBudget:
+    """Tracks token usage against an optional budget."""
+
+    total: int | None = None
+    _spent: int = field(default=0, init=False)
+    _lock: threading.Lock = field(default_factory=threading.Lock, init=False, repr=False)
+
+    def spent(self) -> int:
+        with self._lock:
+            return self._spent
+
+    def remaining(self) -> int | None:
+        if self.total is None:
+            return None
+        with self._lock:
+            return max(0, self.total - self._spent)
+
+    def add(self, tokens: int) -> None:
+        with self._lock:
+            self._spent += tokens
+
+    def exceeded(self) -> bool:
+        if self.total is None:
+            return False
+        with self._lock:
+            return self._spent >= self.total
+
+    def to_dict(self) -> dict[str, int | None]:
+        return {
+            "total": self.total,
+            "spent": self.spent(),
+            "remaining": self.remaining(),
+        }

pi_dynamic_workflows/builtin_workflows.py

@@ -0,0 +1,344 @@
+"""Built-in workflow patterns (5 curated patterns from upstream)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+DEFAULT_MULTI_PERSPECTIVES = [
+    "technical",
+    "product",
+    "security",
+    "user experience",
+    "maintainability",
+]
+
+
+@dataclass
+class BuiltinWorkflowInvocation:
+    script: str
+    name: str
+
+
+@dataclass
+class BuiltinWorkflowDescriptor:
+    name: str
+    description: str
+    resolve: Any  # (args) -> BuiltinWorkflowInvocation
+
+
+def _require_str(args: dict[str, Any], key: str, pattern_name: str) -> str:
+    v = args.get(key)
+    if not isinstance(v, str) or not v.strip():
+        raise ValueError(
+            f'Built-in workflow "{pattern_name}" requires args.{key} to be a non-empty string.'
+        )
+    return v
+
+
+def _require_str_list(args: dict[str, Any], key: str, pattern_name: str) -> list[str]:
+    v = args.get(key)
+    if not isinstance(v, list) or not v or not all(isinstance(s, str) and s.strip() for s in v):
+        raise ValueError(
+            f'Built-in workflow "{pattern_name}" requires args.{key} '
+            "to be a non-empty list of non-empty strings."
+        )
+    return v
+
+
+def _as_record(args: Any) -> dict[str, Any]:
+    return args if isinstance(args, dict) else {}
+
+
+# ---------------------------------------------------------------------------
+# Pattern generators — produce Python workflow scripts
+# ---------------------------------------------------------------------------
+
+
+def generate_deep_research() -> str:
+    return """\
+meta = {"name": "deep_research", "description": "Research a question with cross-checked sources"}
+
+async def main():
+    question = args["question"]
+    phase("Generate search angles")
+    angles = await agent(
+        f"Generate 5 diverse search angles for researching: {question}\\n"
+        "Return each angle on a new line, numbered 1-5.",
+        tier="small",
+    )
+
+    angle_list = [line.strip() for line in (angles or "").split("\\n") if line.strip() and line.strip()[0].isdigit()]
+    if not angle_list:
+        angle_list = [question]
+
+    phase("Research")
+    findings = await parallel([
+        lambda a=angle: agent(
+            f"Research this angle thoroughly: {a}\\n"
+            f"Original question: {question}\\n"
+            "Provide detailed findings with specific facts, data, and sources.",
+            tier="medium",
+        )
+        for angle in angle_list[:5]
+    ])
+
+    phase("Cross-check")
+    combined = "\\n\\n---\\n\\n".join(f"Angle: {a}\\nFindings: {f}" for a, f in zip(angle_list, findings) if f)
+    verified = await agent(
+        f"Cross-check these research findings for accuracy and consistency:\\n\\n{combined}\\n\\n"
+        "Flag any contradictions, unsupported claims, or areas needing more evidence. "
+        "Rate confidence for each finding.",
+        tier="big",
+    )
+
+    phase("Synthesize")
+    report = await agent(
+        f"Synthesize a comprehensive research report on: {question}\\n\\n"
+        f"Research findings:\\n{combined}\\n\\n"
+        f"Cross-check results:\\n{verified}\\n\\n"
+        "Write a well-structured report with an executive summary, key findings, "
+        "confidence levels, and areas for further research.",
+        tier="big",
+    )
+
+    result(report)
+"""
+
+
+def generate_adversarial_review() -> str:
+    return """\
+meta = {"name": "adversarial_review", "description": "Investigate then cross-check with skeptical reviewers"}
+
+async def main():
+    task = args["task"]
+    reviewers = args.get("reviewers", 3)
+
+    phase("Investigate")
+    investigation = await agent(
+        f"Thoroughly investigate: {task}\\n"
+        "Provide detailed findings with evidence and reasoning.",
+        tier="big",
+    )
+
+    phase("Adversarial review")
+    reviews = await parallel([
+        lambda i=i: agent(
+            f"You are Reviewer #{i+1}. Critically review this investigation:\\n\\n{investigation}\\n\\n"
+            "Be skeptical. Challenge assumptions, find gaps, identify weak evidence, "
+            "and suggest what was missed. Rate each finding as: confirmed / questionable / refuted.",
+            tier="medium",
+        )
+        for i in range(reviewers)
+    ])
+
+    phase("Synthesize")
+    all_reviews = "\\n\\n---\\n\\n".join(f"Reviewer #{i+1}:\\n{r}" for i, r in enumerate(reviews) if r)
+    synthesis = await agent(
+        f"Synthesize the investigation and all reviewer feedback:\\n\\n"
+        f"Original investigation:\\n{investigation}\\n\\n"
+        f"Reviews:\\n{all_reviews}\\n\\n"
+        "Produce a final report with confidence-rated findings. "
+        "Only include findings that survived adversarial review.",
+        tier="big",
+    )
+    result(synthesis)
+"""
+
+
+def generate_code_review() -> str:
+    return """\
+meta = {"name": "code_review", "description": "Multi-angle parallel code review"}
+
+async def main():
+    diff = args["diff"]
+    diff_source = args.get("diff_source", "unknown")
+
+    phase("Parallel review")
+    angles = [
+        ("correctness", "Find bugs, logic errors, off-by-one mistakes, race conditions, and unhandled edge cases."),
+        ("security", "Find security vulnerabilities: injection, auth bypass, data exposure, unsafe deserialization."),
+        ("performance", "Find performance issues: N+1 queries, unnecessary allocations, missing caching, O(n²) loops."),
+        ("simplification", "Find unnecessary complexity: dead code, over-abstraction, code that could be simplified."),
+        ("reuse", "Find code duplication and opportunities to use existing utilities or extract shared helpers."),
+    ]
+
+    findings = await parallel([
+        lambda name=name, instruction=instruction: agent(
+            f"Review this diff as a {name} specialist:\\n\\n```diff\\n{diff[:20000]}\\n```\\n\\n{instruction}\\n"
+            "List each finding with file, line, severity (critical/major/minor), and a fix suggestion.",
+            tier="medium",
+        )
+        for name, instruction in angles
+    ])
+
+    phase("Verify and rank")
+    combined = "\\n\\n---\\n\\n".join(
+        f"**{name}** reviewer:\\n{f}"
+        for (name, _), f in zip(angles, findings)
+        if f
+    )
+    verified = await agent(
+        f"You received findings from {len(angles)} code reviewers. Verify and rank them:\\n\\n{combined}\\n\\n"
+        "Remove duplicates and false positives. Rank remaining findings by impact. "
+        "Output a numbered list, most critical first, with actionable fix suggestions.",
+        tier="big",
+    )
+    result(verified)
+"""
+
+
+def generate_multi_perspective(topic: str, perspectives: list[str]) -> str:
+    persp_list = repr(perspectives)
+    return f"""\
+meta = {{"name": "multi_perspective", "description": "Analyze from multiple perspectives"}}
+
+async def main():
+    topic = args.get("topic", {topic!r})
+    perspectives = args.get("perspectives", {persp_list})
+
+    phase("Parallel analysis")
+    analyses = await parallel([
+        lambda p=p: agent(
+            f"Analyze this topic from a {{p}} perspective:\\n\\n{{topic}}\\n\\n"
+            f"Focus exclusively on {{p}} concerns, trade-offs, and implications. "
+            "Be specific and provide actionable insights.",
+            tier="medium",
+        )
+        for p in perspectives
+    ])
+
+    phase("Synthesize")
+    combined = "\\n\\n---\\n\\n".join(
+        f"**{{p}}** perspective:\\n{{a}}"
+        for p, a in zip(perspectives, analyses)
+        if a
+    )
+    synthesis = await agent(
+        f"Synthesize these multi-perspective analyses into a unified assessment:\\n\\n{{combined}}\\n\\n"
+        "Identify consensus, tensions, and trade-offs. "
+        "Provide a balanced recommendation that accounts for all perspectives.",
+        tier="big",
+    )
+    result(synthesis)
+"""
+
+
+def generate_codebase_audit(scope: str, checks: list[str]) -> str:
+    return f"""\
+meta = {{"name": "codebase_audit", "description": "Parallel codebase audit"}}
+
+async def main():
+    scope = args.get("scope", {scope!r})
+    checks = args.get("checks", {checks!r})
+
+    phase("Parallel checks")
+    findings = await parallel([
+        lambda check=check: agent(
+            f"Audit the codebase (scope: {{scope}}) for: {{check}}\\n\\n"
+            "Examine the code thoroughly. List each finding with file path, "
+            "line numbers, severity, and a recommended fix.",
+            tier="medium",
+        )
+        for check in checks
+    ])
+
+    phase("Cross-validate")
+    combined = "\\n\\n---\\n\\n".join(
+        f"**{{check}}**:\\n{{f}}"
+        for check, f in zip(checks, findings)
+        if f
+    )
+    validated = await agent(
+        f"Cross-validate and compile these audit findings:\\n\\n{{combined}}\\n\\n"
+        "Remove false positives, merge overlapping findings, and produce "
+        "a prioritized report with actionable recommendations.",
+        tier="big",
+    )
+    result(validated)
+"""
+
+
+# ---------------------------------------------------------------------------
+# Registry
+# ---------------------------------------------------------------------------
+
+
+def _resolve_multi_perspective(args: Any) -> BuiltinWorkflowInvocation:
+    a = _as_record(args)
+    return BuiltinWorkflowInvocation(
+        script=generate_multi_perspective(
+            _require_str(a, "topic", "multi-perspective"),
+            a.get("perspectives", DEFAULT_MULTI_PERSPECTIVES),
+        ),
+        name="multi-perspective",
+    )
+
+
+def _resolve_codebase_audit(args: Any) -> BuiltinWorkflowInvocation:
+    a = _as_record(args)
+    return BuiltinWorkflowInvocation(
+        script=generate_codebase_audit(
+            _require_str(a, "scope", "codebase-audit"),
+            _require_str_list(a, "checks", "codebase-audit"),
+        ),
+        name="codebase-audit",
+    )
+
+
+BUILTIN_WORKFLOWS: dict[str, BuiltinWorkflowDescriptor] = {
+    "deep-research": BuiltinWorkflowDescriptor(
+        name="deep-research",
+        description=(
+            "Research a question across the web with "
+            "cross-checked sources. args: { question: str }."
+        ),
+        resolve=lambda args: BuiltinWorkflowInvocation(
+            script=generate_deep_research(),
+            name="deep-research",
+        ),
+    ),
+    "adversarial-review": BuiltinWorkflowDescriptor(
+        name="adversarial-review",
+        description=(
+            "Investigate then cross-check with skeptical reviewers. "
+            "args: { task: str, reviewers?: int }."
+        ),
+        resolve=lambda args: BuiltinWorkflowInvocation(
+            script=generate_adversarial_review(),
+            name="adversarial-review",
+        ),
+    ),
+    "code-review": BuiltinWorkflowDescriptor(
+        name="code-review",
+        description="Multi-angle parallel code review. args: { diff: str }.",
+        resolve=lambda args: BuiltinWorkflowInvocation(
+            script=generate_code_review(),
+            name="code-review",
+        ),
+    ),
+    "multi-perspective": BuiltinWorkflowDescriptor(
+        name="multi-perspective",
+        description=(
+            "Analyze a topic from several perspectives, then synthesize. "
+            "args: { topic: str, perspectives?: list[str] }."
+        ),
+        resolve=_resolve_multi_perspective,
+    ),
+    "codebase-audit": BuiltinWorkflowDescriptor(
+        name="codebase-audit",
+        description=(
+            "Run parallel checks against a codebase scope. args: { scope: str, checks: list[str] }."
+        ),
+        resolve=_resolve_codebase_audit,
+    ),
+}
+
+BUILTIN_WORKFLOW_NAMES = list(BUILTIN_WORKFLOWS.keys())
+
+
+def resolve_builtin_workflow(name: str, args: Any = None) -> BuiltinWorkflowInvocation | None:
+    desc = BUILTIN_WORKFLOWS.get(name)
+    if desc is None:
+        return None
+    return desc.resolve(args)

pi_dynamic_workflows/journal.py

@@ -0,0 +1,154 @@
+"""Append-only JSONL journal for workflow run replay.
+
+Records each host call (``agent()``, ``phase()``, ``log()``) with a SHA-256
+request hash so that re-running the same workflow can skip LLM calls whose
+inputs have not changed.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import time
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+MAX_ENTRIES = 10_000
+MAX_FILE_BYTES = 64 * 1024 * 1024  # 64 MB
+
+_MISS = object()
+
+
+def _deterministic_json(obj: Any) -> str:
+    """JSON-serialise *obj* with sorted keys for stable hashing."""
+    return json.dumps(obj, sort_keys=True, ensure_ascii=False, default=str)
+
+
+def hash_request(kind: str, params: Any) -> str:
+    """SHA-256 hex digest of ``kind`` + deterministic JSON of *params*."""
+    payload = _deterministic_json({"kind": kind, "params": params})
+    return hashlib.sha256(payload.encode()).hexdigest()
+
+
+@dataclass
+class JournalEntry:
+    seq: int
+    kind: str
+    req_hash: str
+    result: Any
+    at_ms: int = field(default_factory=lambda: int(time.time() * 1000))
+
+
+class Journal:
+    """Append-only JSONL journal for deterministic replay.
+
+    * ``try_replay(kind, req_hash)`` — returns cached result on hash match,
+      or ``_MISS`` on divergence (auto-truncates).
+    * ``append(kind, req_hash, result)`` — records a new entry.
+    """
+
+    def __init__(self, path: Path | None = None) -> None:
+        self._entries: list[JournalEntry] = []
+        self._path = path
+        self._cursor = 0
+
+    # -- persistence ---------------------------------------------------------
+
+    @classmethod
+    def load(cls, path: Path) -> Journal:
+        """Load an existing journal file (one JSON object per line)."""
+        j = cls(path)
+        if not path.exists():
+            return j
+        try:
+            with open(path, encoding="utf-8") as fh:
+                for line in fh:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    raw = json.loads(line)
+                    j._entries.append(
+                        JournalEntry(
+                            seq=raw["seq"],
+                            kind=raw["kind"],
+                            req_hash=raw["req_hash"],
+                            result=raw.get("result"),
+                            at_ms=raw.get("at_ms", 0),
+                        )
+                    )
+        except (json.JSONDecodeError, KeyError, OSError) as exc:
+            logger.warning("Journal load failed (%s); starting fresh", exc)
+            j._entries.clear()
+        return j
+
+    def _flush_to_disk(self) -> None:
+        if self._path is None:
+            return
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        with open(self._path, "w", encoding="utf-8") as fh:
+            for entry in self._entries:
+                fh.write(json.dumps(asdict(entry), ensure_ascii=False) + "\n")
+
+    # -- replay / record -----------------------------------------------------
+
+    def try_replay(self, kind: str, req_hash: str) -> Any:
+        """Return cached result if the next entry matches, else ``_MISS``.
+
+        On hash mismatch the journal is truncated from the current cursor
+        position so subsequent calls will record fresh entries.
+        """
+        if self._cursor >= len(self._entries):
+            return _MISS
+        entry = self._entries[self._cursor]
+        if entry.kind == kind and entry.req_hash == req_hash:
+            self._cursor += 1
+            return entry.result
+        self.truncate_from(self._cursor)
+        return _MISS
+
+    def append(self, kind: str, req_hash: str, result: Any) -> None:
+        """Record a new host-call result."""
+        if len(self._entries) >= MAX_ENTRIES:
+            logger.warning("Journal entry cap (%d) reached; skipping append", MAX_ENTRIES)
+            return
+        entry = JournalEntry(
+            seq=len(self._entries),
+            kind=kind,
+            req_hash=req_hash,
+            result=result,
+        )
+        line = json.dumps(asdict(entry), ensure_ascii=False) + "\n"
+        if self._path is not None:
+            self._path.parent.mkdir(parents=True, exist_ok=True)
+            current_size = self._path.stat().st_size if self._path.exists() else 0
+            if current_size + len(line.encode("utf-8")) > MAX_FILE_BYTES:
+                logger.warning(
+                    "Journal file size limit (%d bytes) reached; skipping append",
+                    MAX_FILE_BYTES,
+                )
+                return
+            with open(self._path, "a", encoding="utf-8") as fh:
+                fh.write(line)
+        self._entries.append(entry)
+        self._cursor = len(self._entries)
+
+    def truncate_from(self, seq: int) -> None:
+        """Discard entries from *seq* onwards (divergence)."""
+        if seq < len(self._entries):
+            self._entries = self._entries[:seq]
+            self._cursor = seq
+            self._flush_to_disk()
+
+    # -- introspection -------------------------------------------------------
+
+    @property
+    def entry_count(self) -> int:
+        return len(self._entries)
+
+    @property
+    def replayed_count(self) -> int:
+        return self._cursor