devsper 2.1.6__py3-none-any.whl

devsper/workflow/validator.py

@@ -0,0 +1,128 @@
+"""Validate workflow definition: references, DAG, conditions, dead output."""
+
+import re
+from dataclasses import dataclass
+
+from devsper.workflow.resolver import build_execution_order, validate_dag
+from devsper.workflow.schema import WorkflowDefinition, WorkflowStep
+
+
+@dataclass
+class ValidationReport:
+    valid: bool
+    errors: list[str]  # blockers
+    warnings: list[str]  # non-fatal issues
+    info: list[str]  # stats/notes
+
+
+def _template_refs(template: str) -> list[tuple[str, str]]:
+    """Return list of (step_id, field) for references like steps.STEP_ID.FIELD or steps.STEP_ID.result."""
+    refs: list[tuple[str, str]] = []
+    # Match {steps.STEP_ID.FIELD} or {steps.STEP_ID.result}
+    for m in re.finditer(r"\{steps\.([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z0-9_]+)\}", template):
+        refs.append((m.group(1), m.group(2)))
+    return refs
+
+
+def _condition_ref(expression: str) -> tuple[str, str] | None:
+    """Return (step_id, field) if expression is steps.STEP_ID.FIELD <op> value."""
+    m = re.match(
+        r"^\s*steps\.([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_][a-zA-Z0-9_]*)\s",
+        expression.strip(),
+    )
+    if m:
+        return (m.group(1), m.group(2))
+    return None
+
+
+def validate_workflow(definition: WorkflowDefinition) -> ValidationReport:
+    errors: list[str] = []
+    warnings: list[str] = []
+    info: list[str] = []
+
+    step_ids = {s.id for s in definition.steps}
+    step_by_id = {s.id: s for s in definition.steps}
+
+    # DAG and depends_on
+    dag_errors = validate_dag(definition.steps)
+    errors.extend(dag_errors)
+
+    waves = None
+    if not dag_errors:
+        try:
+            waves = build_execution_order(definition.steps)
+            info.append(f"Total steps: {len(definition.steps)}")
+            info.append(f"Parallel waves: {len(waves)}")
+            info.append(f"Critical path length: {len(waves)}")
+        except Exception as e:
+            errors.append(str(e))
+
+    # All {steps.X.field} template references point to steps with output_schema defining that field (or .result)
+    for step in definition.steps:
+        for step_id, field in _template_refs(step.task):
+            if step_id not in step_ids:
+                errors.append(
+                    f"Step {step.id!r} task references steps.{step_id}.{field} but step {step_id!r} does not exist."
+                )
+            else:
+                ref_step = step_by_id[step_id]
+                if field == "result":
+                    continue  # always valid
+                if not ref_step.output_schema:
+                    errors.append(
+                        f"Step {step.id!r} task references steps.{step_id}.{field} but step {step_id!r} has no output_schema."
+                    )
+                else:
+                    names = [f.name for f in ref_step.output_schema]
+                    if field not in names:
+                        errors.append(
+                            f"Step {step.id!r} task references steps.{step_id}.{field} but step {step_id!r} output_schema has: {names}."
+                        )
+
+    # All if: expressions reference steps that appear earlier in dependency order
+    # and output_schema fields referenced in conditions exist
+    if waves:
+        order_so_far: set[str] = set()
+        for wave in waves:
+            for s in wave:
+                if s.if_:
+                    ref = _condition_ref(s.if_.expression)
+                    if ref:
+                        cond_step_id, cond_field = ref
+                        if cond_step_id not in order_so_far:
+                            errors.append(
+                                f"Step {s.id!r} condition references steps.{cond_step_id}.{cond_field} "
+                                f"but {cond_step_id!r} is not a dependency (must run before this step)."
+                            )
+                        elif cond_step_id in step_by_id:
+                            cond_step = step_by_id[cond_step_id]
+                            if cond_field != "result":
+                                names = [f.name for f in cond_step.output_schema]
+                                if cond_field not in names:
+                                    errors.append(
+                                        f"Step {s.id!r} condition references steps.{cond_step_id}.{cond_field} "
+                                        f"but that step's output_schema has: {names}."
+                                    )
+                order_so_far.add(s.id)
+
+    # Warning: steps with no id set (we require id; no auto-generated opaque ids in schema)
+    # Our schema requires id, so skip.
+
+    # Warning: steps with output_schema but no downstream steps that use the output (dead output)
+    consumers: dict[str, set[str]] = {sid: set() for sid in step_ids}
+    for step in definition.steps:
+        for step_id, field in _template_refs(step.task):
+            if step_id in step_ids:
+                consumers[step_id].add(step.id)
+    for step in definition.steps:
+        if step.output_schema and step.id in consumers and not consumers[step.id]:
+            warnings.append(
+                f"Step {step.id!r} has output_schema but no downstream step uses its output (dead output)."
+            )
+
+    return ValidationReport(
+        valid=len(errors) == 0,
+        errors=errors,
+        warnings=warnings,
+        info=info,
+    )

devsper-2.1.6.dist-info/METADATA

@@ -0,0 +1,346 @@
+Metadata-Version: 2.4
+Name: devsper
+Version: 2.1.6
+Summary: Orchestrate distributed swarms of AI agents that collaboratively solve complex tasks.
+Project-URL: Homepage, https://github.com/devsper-com/runtime
+Project-URL: Documentation, https://docs.devsper.com
+Project-URL: Changelog, https://github.com/devsper-com/runtime/blob/main/CHANGELOG.md
+Project-URL: Worker, https://github.com/devsper-com/runtime/pkgs/container/devsper-worker
+License-Expression: GPL-3.0-or-later
+License-File: LICENSE
+Keywords: agents,ai,distributed,llm,multi-agent,orchestration,swarm
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.12
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: keyring>=24.0
+Requires-Dist: langchain-anthropic>=1.3.4
+Requires-Dist: langchain-google-genai>=4.2.1
+Requires-Dist: langchain-openai>=1.1.10
+Requires-Dist: langchain>=1.2.10
+Requires-Dist: networkx>=3.6.1
+Requires-Dist: numpy>=2.4.2
+Requires-Dist: pydantic>=2.12.5
+Requires-Dist: pyfiglet>=1.0
+Requires-Dist: python-dotenv>=1.2.2
+Requires-Dist: rich>=14.3.3
+Requires-Dist: shtab>=1.6.0
+Requires-Dist: textual>=0.50.0
+Provides-Extra: a2a
+Requires-Dist: fastapi>=0.110; extra == 'a2a'
+Requires-Dist: httpx>=0.27; extra == 'a2a'
+Requires-Dist: sse-starlette>=2.0; extra == 'a2a'
+Requires-Dist: uvicorn>=0.29; extra == 'a2a'
+Provides-Extra: compliance
+Requires-Dist: spacy>=3.7; extra == 'compliance'
+Provides-Extra: data
+Requires-Dist: pandas>=2.0.0; extra == 'data'
+Requires-Dist: scikit-learn>=1.3.0; extra == 'data'
+Provides-Extra: distributed
+Requires-Dist: fastapi>=0.110.0; extra == 'distributed'
+Requires-Dist: hiredis>=2.3.0; extra == 'distributed'
+Requires-Dist: redis>=5.0.0; extra == 'distributed'
+Requires-Dist: uvicorn>=0.29.0; extra == 'distributed'
+Provides-Extra: docs
+Requires-Dist: mike>=1.1.0; extra == 'docs'
+Requires-Dist: mkdocs-material>=9.5.0; extra == 'docs'
+Requires-Dist: mkdocs<2,>=1.6.0; extra == 'docs'
+Provides-Extra: document
+Requires-Dist: docproc>=2.0.0; extra == 'document'
+Requires-Dist: python-docx>=1.0.0; extra == 'document'
+Provides-Extra: embeddings
+Requires-Dist: sentence-transformers>=3.0.0; extra == 'embeddings'
+Provides-Extra: explainability
+Provides-Extra: hitl
+Requires-Dist: httpx>=0.27; extra == 'hitl'
+Provides-Extra: mcp
+Requires-Dist: anyio>=4.0; extra == 'mcp'
+Requires-Dist: httpx>=0.27; extra == 'mcp'
+Provides-Extra: ollama
+Provides-Extra: research
+Requires-Dist: duckduckgo-search>=6.0.0; extra == 'research'
+Provides-Extra: worker
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/rithulkamesh/devsper/refs/heads/main/branding/logo_bg.svg" alt="devsper" width="120" height="140" />
+</p>
+
+<h1 align="center">devsper</h1>
+<p align="center"><strong>Distributed AI Swarm Runtime</strong></p>
+
+<p align="center">
+  <a href="https://pypi.org/project/devsper/"><img src="https://img.shields.io/pypi/v/devsper?label=PyPI" alt="PyPI"></a>
+  <a href="https://www.gnu.org/licenses/gpl-3.0"><img src="https://img.shields.io/badge/License-GPLv3-blue.svg" alt="License: GPL v3"></a>
+  <a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.12+-green.svg" alt="Python 3.12+"></a>
+</p>
+
+<p align="center">
+  <em>Orchestrate multi-agent systems with a swarm execution model: tasks → DAG → parallel execution.</em>
+</p>
+
+> **Install:** PyPI package **`devsper`** · CLI **`devsper`**
+
+---
+
+## Quick start
+
+**1. Install (Python 3.12+):**
+
+```bash
+pip install devsper
+# or: uv add devsper
+```
+
+**2. Set up API keys (pick one):**
+
+Store credentials in your OS keychain so you never re-enter them:
+
+```bash
+devsper credentials set openai api_key      # prompts for value
+devsper credentials set anthropic api_key
+devsper credentials set github token
+# or migrate from .env:
+devsper credentials migrate
+```
+
+Or use environment variables: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GITHUB_TOKEN`, etc. (see [Credentials](#credentials)).
+
+**3. Create a project and run:**
+
+```bash
+devsper init
+devsper run "Summarize swarm intelligence in one paragraph."
+```
+
+**4. Optional — shell completion:**
+
+```bash
+# Bash: add to ~/.bashrc
+eval "$(devsper completion bash)"
+
+# Zsh: add to ~/.zshrc
+eval "$(devsper completion zsh)"
+```
+
+---
+
+## Run from code
+
+**From config file:**
+
+```python
+from devsper import Swarm
+
+swarm = Swarm(config="devsper.toml")
+results = swarm.run("Analyze diffusion models and write a one-page summary.")
+```
+
+**Explicit parameters:**
+
+```python
+from devsper import Swarm
+
+swarm = Swarm(worker_count=4, worker_model="gpt-4o-mini", planner_model="gpt-4o-mini", use_tools=True)
+results = swarm.run("Your task here.")
+```
+
+Credentials are injected from the keyring (or env) when config is resolved—no code changes needed.
+
+---
+
+## Credentials
+
+API keys are **not** stored in config files. Use the **credential store** (OS keychain) or environment variables.
+
+| What you want | Command or method |
+|---------------|-------------------|
+| Store a key securely | `devsper credentials set <provider> <key>` (prompts; uses keyring) |
+| List stored keys (no values) | `devsper credentials list` |
+| Import from `.env` / TOML | `devsper credentials migrate` |
+| Export for sourcing / `.env` | `devsper credentials export <provider>` → prints `KEY=value` lines |
+| Remove a key | `devsper credentials delete <provider> <key>` |
+
+**Providers:** `openai`, `anthropic`, `github`, `gemini`, `azure`, `azure_anthropic` (keys: `api_key`, `token`, `endpoint`, `deployment`, `api_version` as applicable).
+
+**Example — export and source in a script:**
+
+```bash
+eval "$(devsper credentials export azure)"
+devsper run "Your task"
+```
+
+See [Configuration](docs/configuration.md#credentials-api-keys) and [CLI](docs/cli.md#credentials) for details.
+
+---
+
+## CLI
+
+| Command | Description |
+|--------|-------------|
+| `devsper init` | Set up a new project (`devsper.toml`) |
+| `devsper doctor` | Check environment (keys, config, tools) |
+| `devsper run "task"` | Run swarm on a task |
+| `devsper tui` | Terminal UI (prompt, dashboard, logs) |
+| `devsper credentials set/list/migrate/export/delete` | Manage API keys (keyring) |
+| `devsper completion bash \| zsh` | Print shell completion script |
+| `devsper research [path]` | Literature review on a directory |
+| `devsper analyze [path]` | Analyze repository architecture |
+| `devsper memory [--limit N]` | List memory entries |
+| `devsper query "…"` | Query knowledge graph |
+| `devsper workflow <name>` | Run a workflow from `workflow.devsper.toml` |
+| `devsper graph [run_id]` | Export task DAG as Mermaid |
+| `devsper replay [run_id]` | Replay a run from event log |
+| `devsper cache stats \| clear` | Task result cache |
+| `devsper analytics` | Tool usage stats |
+| `devsper build "app description" [-o dir]` | Autonomous app builder |
+| `devsper upgrade [--check \| -y]` | Check for updates / upgrade |
+
+Run `devsper --help` or `devsper <command> --help` for examples and options.
+
+---
+
+## Features
+
+- **Planner → Scheduler → Executor → Agents** — DAG-based execution with configurable parallelism
+- **Strategy-based planning** — Auto-selected strategies (research, code, data science, document, experiment) or LLM fallback
+- **120+ tools** — Research, coding, data science, documents, experiments, memory; **smart tool selection** (top-k by similarity)
+- **TOML config** — `devsper.toml` / `workflow.devsper.toml`; env > project > user > defaults
+- **Memory & knowledge graph** — Episodic, semantic, research, artifact memory; summarization, namespaces, entity/relationship search
+- **Map-reduce runtime** — `swarm.map_reduce(dataset, map_fn, reduce_fn)` using the worker pool
+- **Workflows** — Define steps in `workflow.devsper.toml`; run with `devsper workflow <name>`; **structured output self-correction** (v1.7) retries with a correction prompt when JSON parsing fails
+- **Critic & agent messaging (v1.7)** — Optional second-pass critic scores results and requests one retry; per-run message bus lets agents share discoveries via `BROADCAST:`
+- **Speculative pre-fetching (v1.7)** — Pre-warm memory and tools for successor tasks while others run; reduces standing-up time
+- **Plugin ecosystem** — Discover tools via entry_points (`devsper.plugins`)
+- **Provider routing** — OpenAI, Anthropic, Azure, Gemini, **GitHub Models (Copilot)** (`provider:model` or model name); **429 retry with backoff** for GitHub rate limits
+- **Automatic model routing** — `planner = "auto"` and `worker = "auto"` for cost/latency/quality-aware selection
+- **EventLog, replay, telemetry** — Structured events for debugging and metrics
+
+---
+
+## Architecture
+
+```
+    Planner
+       ↓
+    Scheduler
+       ↓
+    Executor
+       ↓
+    Agents  →  Tools  →  Memory  →  Knowledge Graph
+```
+
+---
+
+## Configuration
+
+**Priority:** env > project config > user `~/.config/devsper/config.toml` > defaults.
+
+**Locations:** `./devsper.toml`, `./workflow.devsper.toml`, `~/.config/devsper/config.toml`, or legacy `.devsper/config.toml`.
+
+**Keep secrets out of TOML.** Use `devsper credentials` or environment variables for API keys. Non-secret settings (models, workers, paths) go in TOML.
+
+**Example `devsper.toml`:**
+
+```toml
+[swarm]
+workers = 6
+adaptive_planning = true
+max_iterations = 10
+critic_enabled = true
+critic_roles = ["research", "analysis", "code"]
+message_bus_enabled = true
+prefetch_enabled = true
+
+[models]
+planner = "auto"
+worker = "auto"
+
+[memory]
+enabled = true
+store_results = true
+top_k = 5
+
+[tools]
+enabled = ["research", "coding", "documents"]
+top_k = 12
+
+[telemetry]
+enabled = true
+save_events = true
+
+[providers.azure]
+endpoint = ""   # or use credentials store / env
+deployment = ""
+```
+
+Env overrides: `DEVSPER_WORKER_MODEL`, `DEVSPER_PLANNER_MODEL`, `DEVSPER_EVENTS_DIR`, `DEVSPER_DATA_DIR`, plus provider keys. Full schema: [docs/configuration.md](docs/configuration.md), [docs/providers.md](docs/providers.md).
+
+---
+
+## Distributed mode (v1.10)
+
+Run a **controller** and **workers** across processes or machines. Workers can be Python or **Rust** (`devsper-worker` binary) for higher throughput.
+
+```bash
+# Redis + workers + controller (see examples/distributed/README.md)
+docker compose up -d
+uv run python examples/distributed/run_worker.py   # or Rust: DEVSPER_WORKER_MODEL=github:gpt-4o ./worker/target/release/devsper-worker
+uv run python examples/distributed/run_controller.py "Your task" --parallel
+```
+
+Rust workers: set `DEVSPER_WORKER_MODEL=github:gpt-4o` (or your model), `DEVSPER_PYTHON_BIN=.venv/bin/python`, `DEVSPER_RPC_PORT=0` for multiple workers on one host. Credentials load from keychain in the subprocess.
+
+---
+
+## Examples
+
+| Workflow | Command |
+|----------|---------|
+| Distributed (v1.10) | `uv run python examples/distributed/run_controller.py "Task" --parallel` |
+| Literature review | `devsper research papers/` or `uv run python examples/research/literature_review.py [dir]` |
+| Repository analysis | `devsper analyze .` or `uv run python examples/coding/analyze_repository.py [path]` |
+| Dataset analysis | `uv run python examples/data_science/dataset_analysis.py [path-to.csv]` |
+| Document intelligence | `uv run python examples/documents/analyze_documents.py [dir]` |
+| Parameter sweep | `uv run python examples/experiments/parameter_sweep.py --params '{"lr":[0.01,0.1]}'` |
+
+Outputs under `examples/output/`. Run from project root when using script paths.
+
+---
+
+## Documentation
+
+Full docs (with versioning and dark mode): **[docs.devsper.com](https://docs.devsper.com)**. Source lives in `website/docs/` and is built with [Docusaurus](https://docusaurus.io).
+
+| Doc | Description |
+|-----|-------------|
+| [Introduction](https://docs.devsper.com/docs/introduction) | What devsper is, problem, core concepts |
+| [Architecture](https://docs.devsper.com/docs/architecture) | Planner, Scheduler, Executor, Agents, Tools, Memory, strategies |
+| [Configuration](https://docs.devsper.com/docs/configuration) | TOML schema, locations, env, **credentials** |
+| [Swarm runtime](https://docs.devsper.com/docs/swarm_runtime) | Task lifecycle, flow, map-reduce |
+| [Tools](https://docs.devsper.com/docs/tools) | Registry, runner, smart selection, plugins |
+| [Memory](https://docs.devsper.com/docs/memory_system) | Types, store, retrieval, knowledge graph |
+| [Providers](https://docs.devsper.com/docs/providers) | Provider routing, Azure, GitHub Models, auto routing |
+| [CLI](https://docs.devsper.com/docs/cli) | All commands, **credentials**, completion |
+| [TUI](https://docs.devsper.com/docs/tui) | Layout, panels, shortcuts |
+| [Examples](https://docs.devsper.com/docs/examples) | Workflows and commands |
+| [Development](https://docs.devsper.com/docs/development) | Structure, adding tools/plugins/workflows |
+| [Contributing](CONTRIBUTING.md) | Setup, testing, PR guidelines |
+| [FAQ](https://docs.devsper.com/docs/faq) | Common questions |
+
+---
+
+## Contributing
+
+Contributions welcome. See [CONTRIBUTING.md](CONTRIBUTING.md).
+
+---
+
+## License
+
+**GPL-3.0-or-later** — see [LICENSE](LICENSE).