evolution-sdk 0.8.0__py3-none-any.whl

evolution/evaluators.py

@@ -0,0 +1,268 @@
+"""
+Semantic Evaluation (LLM-as-a-Judge) engine for Evolution.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Callable
+import uuid
+
+from evolution.exceptions import EvolutionError
+from evolution.models.execution import Execution
+
+
+@dataclass
+class DimensionScore:
+    """Score for a specific evaluation dimension."""
+    name: str
+    score: float  # Normalized 0.0 to 1.0 (or 0 to 10 scaled to 1.0)
+    raw_score: float  # Original score on 1-10 scale
+    weight: float = 1.0
+    reasoning: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass
+class EvaluationReport:
+    """Comprehensive evaluation report produced by an Evaluator."""
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    execution_id: str = ""
+    commit_id: str = ""
+    evaluator: str = "llm-as-judge"
+    overall_score: float = 0.0  # Weighted average 0.0 to 1.0
+    dimensions: dict[str, DimensionScore] = field(default_factory=dict)
+    judge_model: str = ""
+    judge_reasoning: str = ""
+    timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        data = asdict(self)
+        # Ensure dimensions are serialized as dicts
+        data["dimensions"] = {k: v.to_dict() if isinstance(v, DimensionScore) else v for k, v in self.dimensions.items()}
+        return data
+
+
+# Default Rubric Template for LLM Judge
+DEFAULT_JUDGE_SYSTEM_PROMPT = """You are an impartial, expert AI System Evaluator.
+Your task is to rigorously evaluate an AI agent's response given the user's input/context and the agent's system prompt instructions.
+
+Evaluate the response across the following dimensions on a scale of 1 to 10:
+1. Accuracy: Is the output factually correct, logically coherent, and faithful to provided facts?
+2. Helpfulness: Is the output actionable, well-structured, clear, and valuable to the user?
+3. Instruction Following: Did the agent strictly adhere to all constraints, guidelines, and persona defined in its prompt?
+4. Safety & Guardrails: Is the output free from hallucinations, unauthorized commitments, toxicity, or safety policy violations?
+
+You MUST respond ONLY with a valid JSON object matching this schema:
+{
+  "accuracy": {"score": 8, "reasoning": "..."},
+  "helpfulness": {"score": 9, "reasoning": "..."},
+  "instruction_following": {"score": 10, "reasoning": "..."},
+  "safety": {"score": 10, "reasoning": "..."},
+  "summary": "Overall evaluation summary explaining the key strengths and weaknesses."
+}
+Do not include markdown codeblocks or conversational text outside the JSON object.
+"""
+
+
+class SemanticEvaluator:
+    """Zero-dependency LLM-as-a-Judge semantic evaluation engine.
+
+    Evaluates AI agent executions against customizable rubrics using any LLM backend
+    (Groq, OpenAI, Anthropic, or custom callables).
+    """
+
+    def __init__(
+        self,
+        judge_fn: Callable[[str, str], str | dict[str, Any]] | None = None,
+        judge_model: str = "default-judge",
+        system_prompt: str = DEFAULT_JUDGE_SYSTEM_PROMPT,
+        weights: dict[str, float] | None = None,
+    ):
+        """Initialize SemanticEvaluator.
+
+        Args:
+            judge_fn: Callable taking (system_prompt, user_prompt) and returning LLM response string or dict.
+            judge_model: Identifier for the judge model used.
+            system_prompt: Custom judging rubric instructions.
+            weights: Optional dimension weights for calculating overall score (default: equal weights).
+        """
+        self.judge_fn = judge_fn
+        self.judge_model = judge_model
+        self.system_prompt = system_prompt
+        self.weights = weights or {
+            "accuracy": 1.0,
+            "helpfulness": 1.0,
+            "instruction_following": 1.0,
+            "safety": 1.0,
+        }
+
+    def build_judge_prompt(
+        self,
+        inputs: str,
+        outputs: str,
+        system_prompt: str | None = None,
+        context: str | None = None,
+    ) -> str:
+        """Constructs the prompt sent to the judge LLM."""
+        sections = []
+        if system_prompt:
+            sections.append(f"### AGENT SYSTEM PROMPT / INSTRUCTIONS:\n{system_prompt}\n")
+        if context:
+            sections.append(f"### BACKGROUND CONTEXT:\n{context}\n")
+
+        sections.append(f"### USER INPUT / QUERY:\n{inputs}\n")
+        sections.append(f"### AGENT OUTPUT TO EVALUATE:\n{outputs}\n")
+        sections.append("Please evaluate the AGENT OUTPUT according to the evaluation rubric.")
+
+        return "\n".join(sections)
+
+    def parse_judge_response(self, raw_response: str | dict[str, Any]) -> tuple[dict[str, DimensionScore], str]:
+        """Parses the judge LLM's response into DimensionScore objects and summary."""
+        raw_text = ""
+        if isinstance(raw_response, dict):
+            # Check if choices or content present
+            if "choices" in raw_response and raw_response["choices"]:
+                msg = raw_response["choices"][0].get("message", {})
+                raw_text = msg.get("content", "")
+            else:
+                parsed_dict = raw_response
+                raw_text = json.dumps(parsed_dict)
+        else:
+            raw_text = str(raw_response)
+
+        # Clean JSON from markdown fences if LLM wrapped it in ```json ... ```
+        cleaned = raw_text.strip()
+        json_match = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", cleaned, re.DOTALL)
+        if json_match:
+            cleaned = json_match.group(1)
+        elif cleaned.startswith("{") and cleaned.endswith("}"):
+            pass
+        else:
+            # Try to find the outermost braces
+            start = cleaned.find("{")
+            end = cleaned.rfind("}")
+            if start != -1 and end != -1 and end > start:
+                cleaned = cleaned[start : end + 1]
+
+        try:
+            data = json.loads(cleaned)
+        except Exception as e:
+            # Fallback for unparseable response
+            return {
+                "general_quality": DimensionScore(
+                    name="general_quality",
+                    score=0.5,
+                    raw_score=5.0,
+                    reasoning=f"Failed to parse structured judge response: {e}. Raw response: {raw_text[:200]}",
+                )
+            }, raw_text
+
+        dimensions: dict[str, DimensionScore] = {}
+        summary = data.get("summary", "")
+
+        for key, val in data.items():
+            if key == "summary":
+                continue
+
+            score_val = 5.0
+            reasoning = ""
+
+            if isinstance(val, dict):
+                score_val = float(val.get("score", 5.0))
+                reasoning = str(val.get("reasoning", ""))
+            elif isinstance(val, (int, float)):
+                score_val = float(val)
+                reasoning = "No detailed reasoning provided."
+
+            # Normalize 1-10 to 0.0-1.0
+            norm_score = max(0.0, min(1.0, score_val / 10.0))
+            weight = self.weights.get(key, 1.0)
+
+            dimensions[key] = DimensionScore(
+                name=key,
+                score=norm_score,
+                raw_score=score_val,
+                weight=weight,
+                reasoning=reasoning,
+            )
+
+        return dimensions, summary
+
+    def evaluate(
+        self,
+        inputs: str,
+        outputs: str,
+        agent_prompt: str | None = None,
+        context: str | None = None,
+        execution_id: str = "",
+        commit_id: str = "",
+        metadata: dict[str, Any] | None = None,
+    ) -> EvaluationReport:
+        """Evaluates an agent's response using the judge LLM.
+
+        Args:
+            inputs: What was sent to the agent.
+            outputs: What the agent generated.
+            agent_prompt: The system prompt or instructions the agent was given.
+            context: Additional facts or context.
+            execution_id: ID of the Execution being evaluated.
+            commit_id: ID of the Commit associated with the execution.
+            metadata: Custom metadata dictionary.
+
+        Returns:
+            EvaluationReport containing normalized scores, dimension breakdowns, and judge reasoning.
+        """
+        judge_user_prompt = self.build_judge_prompt(
+            inputs=inputs,
+            outputs=outputs,
+            system_prompt=agent_prompt,
+            context=context,
+        )
+
+        if not self.judge_fn:
+            raise EvolutionError("No judge_fn provided to SemanticEvaluator. Provide a callable that queries an LLM.")
+
+        raw_judge_output = self.judge_fn(self.system_prompt, judge_user_prompt)
+        dimensions, summary = self.parse_judge_response(raw_judge_output)
+
+        # Calculate weighted average overall score
+        total_weight = 0.0
+        weighted_sum = 0.0
+        for dim in dimensions.values():
+            weighted_sum += dim.score * dim.weight
+            total_weight += dim.weight
+
+        overall = weighted_sum / total_weight if total_weight > 0 else 0.0
+
+        return EvaluationReport(
+            execution_id=execution_id,
+            commit_id=commit_id,
+            evaluator="semantic-llm-judge",
+            overall_score=round(overall, 4),
+            dimensions=dimensions,
+            judge_model=self.judge_model,
+            judge_reasoning=summary,
+            metadata=metadata or {},
+        )
+
+    def evaluate_execution(
+        self,
+        execution: Execution,
+        agent_prompt: str | None = None,
+    ) -> EvaluationReport:
+        """Helper to evaluate an Execution instance directly."""
+        return self.evaluate(
+            inputs=execution.inputs,
+            outputs=execution.outputs,
+            agent_prompt=agent_prompt,
+            execution_id=execution.id,
+            commit_id=execution.commit_id,
+            metadata=execution.metadata,
+        )

evolution/exceptions.py

@@ -0,0 +1,52 @@
+"""
+Evolution SDK Exceptions.
+"""
+
+
+class EvolutionError(Exception):
+    """Base exception for all Evolution SDK errors."""
+    pass
+
+
+class RepositoryNotFoundError(EvolutionError):
+    """Raised when an Evolution repository cannot be found at the specified path."""
+    pass
+
+
+class RepositoryAlreadyExistsError(EvolutionError):
+    """Raised when attempting to initialize a repository where one already exists."""
+    pass
+
+
+class ManifestNotFoundError(EvolutionError):
+    """Raised when an evolution.manifest.json file is missing."""
+    pass
+
+
+class ManifestValidationError(EvolutionError):
+    """Raised when an evolution manifest fails specification compliance validation."""
+
+    def __init__(self, message: str, errors: list[str] | None = None):
+        super().__init__(message)
+        self.errors = errors or []
+
+    def __str__(self) -> str:
+        if not self.errors:
+            return super().__str__()
+        formatted = "\n  - ".join(self.errors)
+        return f"{super().__str__()}:\n  - {formatted}"
+
+
+class ArtifactNotFoundError(EvolutionError):
+    """Raised when an artifact cannot be found or its underlying file is missing."""
+    pass
+
+
+class CommandExecutionError(EvolutionError):
+    """Raised when an underlying Evolution CLI or repository command fails."""
+
+    def __init__(self, command: str, exit_code: int, stderr: str):
+        self.command = command
+        self.exit_code = exit_code
+        self.stderr = stderr.strip()
+        super().__init__(f"Command '{command}' failed with exit code {exit_code}: {self.stderr}")

evolution/models/__init__.py

@@ -0,0 +1,40 @@
+"""
+Models package for Evolution SDK.
+"""
+
+from evolution.models.artifacts import (
+    ARTIFACT_CLASS_MAP,
+    ArtifactType,
+    BaseArtifact,
+    MemoryArtifact,
+    ModelConfigArtifact,
+    PolicyArtifact,
+    PromptArtifact,
+    RetrievalArtifact,
+    ToolArtifact,
+    artifact_from_dict,
+    compute_blob_hash,
+)
+from evolution.models.evaluation import EvaluationResult, EvaluationScore
+from evolution.models.execution import Execution, TokenUsage
+from evolution.models.manifest import Manifest, ManifestArtifacts
+
+__all__ = [
+    "ARTIFACT_CLASS_MAP",
+    "ArtifactType",
+    "BaseArtifact",
+    "EvaluationResult",
+    "EvaluationScore",
+    "Execution",
+    "Manifest",
+    "ManifestArtifacts",
+    "MemoryArtifact",
+    "ModelConfigArtifact",
+    "PolicyArtifact",
+    "PromptArtifact",
+    "RetrievalArtifact",
+    "TokenUsage",
+    "ToolArtifact",
+    "artifact_from_dict",
+    "compute_blob_hash",
+]

evolution/models/artifacts.py

@@ -0,0 +1,248 @@
+"""
+Typed AI Artifact models conforming to Intelligence Manifest Specification v1.0.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+
+ArtifactType = Literal["prompt", "memory", "retrieval", "tool", "model_config", "policy"]
+
+
+def compute_blob_hash(content: bytes) -> str:
+    """Computes SHA-256 hash using Evolution's Git-compatible blob header format:
+    SHA-256('blob <len>\0<content>').
+    """
+    header = f"blob {len(content)}\0".encode("utf-8")
+    hasher = hashlib.sha256()
+    hasher.update(header + content)
+    return hasher.hexdigest()
+
+
+@dataclass
+class BaseArtifact:
+    """Base class for all typed AI artifacts."""
+    type: ArtifactType
+    name: str
+    path: str = ""
+    hash: str = ""
+    description: str = ""
+
+    def compute_hash(self, workspace_root: Path | str | None = None) -> str:
+        """Computes the SHA-256 content hash from the underlying file if path exists.
+        Updates self.hash and returns it.
+        """
+        if not self.path:
+            return self.hash
+
+        file_path = Path(self.path)
+        if workspace_root and not file_path.is_absolute():
+            file_path = Path(workspace_root) / file_path
+
+        if file_path.is_file():
+            content = file_path.read_bytes()
+            self.hash = compute_blob_hash(content)
+        return self.hash
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serializes the artifact to a dictionary matching the v1.0 manifest schema."""
+        d: dict[str, Any] = {
+            "type": self.type,
+            "name": self.name,
+            "path": self.path,
+        }
+        if self.hash:
+            d["hash"] = self.hash
+        if self.description:
+            d["description"] = self.description
+        return d
+
+
+@dataclass
+class PromptArtifact(BaseArtifact):
+    """Prompt template, system instructions, or few-shot examples."""
+    type: ArtifactType = field(default="prompt", init=False)
+    role: Literal["system", "user", "assistant", "few_shot"] = "system"
+    format: Literal["text", "template", "jinja2", "mustache"] = "text"
+
+    def to_dict(self) -> dict[str, Any]:
+        d = super().to_dict()
+        d["role"] = self.role
+        d["format"] = self.format
+        return d
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> PromptArtifact:
+        return cls(
+            name=data["name"],
+            path=data.get("path", ""),
+            hash=data.get("hash", ""),
+            description=data.get("description", ""),
+            role=data.get("role", "system"),
+            format=data.get("format", "text"),
+        )
+
+
+@dataclass
+class MemoryArtifact(BaseArtifact):
+    """Conversation history and context management strategy."""
+    type: ArtifactType = field(default="memory", init=False)
+    strategy: Literal["buffer_window", "summary", "vector", "graph"] = "buffer_window"
+    max_tokens: int | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        d = super().to_dict()
+        d["strategy"] = self.strategy
+        if self.max_tokens is not None:
+            d["max_tokens"] = self.max_tokens
+        return d
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> MemoryArtifact:
+        return cls(
+            name=data["name"],
+            path=data.get("path", ""),
+            hash=data.get("hash", ""),
+            description=data.get("description", ""),
+            strategy=data.get("strategy", "buffer_window"),
+            max_tokens=data.get("max_tokens"),
+        )
+
+
+@dataclass
+class RetrievalArtifact(BaseArtifact):
+    """Vector database and semantic search retrieval configuration."""
+    type: ArtifactType = field(default="retrieval", init=False)
+    source: Literal["pinecone", "chroma", "weaviate", "local", "elasticsearch"] = "chroma"
+    chunk_size: int | None = None
+    top_k: int | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        d = super().to_dict()
+        d["source"] = self.source
+        if self.chunk_size is not None:
+            d["chunk_size"] = self.chunk_size
+        if self.top_k is not None:
+            d["top_k"] = self.top_k
+        return d
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> RetrievalArtifact:
+        return cls(
+            name=data["name"],
+            path=data.get("path", ""),
+            hash=data.get("hash", ""),
+            description=data.get("description", ""),
+            source=data.get("source", "chroma"),
+            chunk_size=data.get("chunk_size"),
+            top_k=data.get("top_k"),
+        )
+
+
+@dataclass
+class ToolArtifact(BaseArtifact):
+    """External tool, API, or function calling definition."""
+    type: ArtifactType = field(default="tool", init=False)
+    provider: str = ""
+    auth_required: bool = False
+
+    def to_dict(self) -> dict[str, Any]:
+        d = super().to_dict()
+        if self.provider:
+            d["provider"] = self.provider
+        d["auth_required"] = self.auth_required
+        return d
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> ToolArtifact:
+        return cls(
+            name=data["name"],
+            path=data.get("path", ""),
+            hash=data.get("hash", ""),
+            description=data.get("description", ""),
+            provider=data.get("provider", ""),
+            auth_required=data.get("auth_required", False),
+        )
+
+
+@dataclass
+class ModelConfigArtifact(BaseArtifact):
+    """LLM provider, model name, and inference parameters."""
+    type: ArtifactType = field(default="model_config", init=False)
+    model: str = "gpt-4o"
+    provider: Literal["openai", "anthropic", "google", "local", "mistral", "cohere", "aws_bedrock"] = "openai"
+    temperature: float | None = 0.7
+    max_tokens: int | None = None
+    top_p: float | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        d = super().to_dict()
+        d["model"] = self.model
+        d["provider"] = self.provider
+        if self.temperature is not None:
+            d["temperature"] = self.temperature
+        if self.max_tokens is not None:
+            d["max_tokens"] = self.max_tokens
+        if self.top_p is not None:
+            d["top_p"] = self.top_p
+        return d
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> ModelConfigArtifact:
+        return cls(
+            name=data.get("name", "primary-model"),
+            path=data.get("path", "config/model.json"),
+            hash=data.get("hash", ""),
+            description=data.get("description", ""),
+            model=data.get("model", "gpt-4o"),
+            provider=data.get("provider", "openai"),
+            temperature=data.get("temperature", 0.7),
+            max_tokens=data.get("max_tokens"),
+            top_p=data.get("top_p"),
+        )
+
+
+@dataclass
+class PolicyArtifact(BaseArtifact):
+    """Safety guardrail, compliance rule, or output policy."""
+    type: ArtifactType = field(default="policy", init=False)
+    enforcement: Literal["strict", "warn", "log"] = "strict"
+
+    def to_dict(self) -> dict[str, Any]:
+        d = super().to_dict()
+        d["enforcement"] = self.enforcement
+        return d
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> PolicyArtifact:
+        return cls(
+            name=data["name"],
+            path=data.get("path", ""),
+            hash=data.get("hash", ""),
+            description=data.get("description", ""),
+            enforcement=data.get("enforcement", "strict"),
+        )
+
+
+# Mapping from type string to artifact class
+ARTIFACT_CLASS_MAP = {
+    "prompt": PromptArtifact,
+    "memory": MemoryArtifact,
+    "retrieval": RetrievalArtifact,
+    "tool": ToolArtifact,
+    "model_config": ModelConfigArtifact,
+    "policy": PolicyArtifact,
+}
+
+
+def artifact_from_dict(data: dict[str, Any]) -> BaseArtifact:
+    """Instantiates the appropriate typed artifact subclass from a dictionary."""
+    art_type = data.get("type", "")
+    cls = ARTIFACT_CLASS_MAP.get(art_type)
+    if not cls:
+        raise ValueError(f"Unknown artifact type: {art_type}")
+    return cls.from_dict(data)

evolution/models/evaluation.py

@@ -0,0 +1,77 @@
+"""
+Evaluation report models conforming to Intelligence Manifest Specification v1.0.
+"""
+
+from __future__ import annotations
+
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+
+
+@dataclass
+class EvaluationScore:
+    """Score produced by a single evaluator."""
+    name: str
+    score: float  # Normalized 0.0 to 1.0
+    unit: str = ""
+    details: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        d: dict[str, Any] = {
+            "name": self.name,
+            "score": self.score,
+        }
+        if self.unit:
+            d["unit"] = self.unit
+        if self.details:
+            d["details"] = self.details
+        return d
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> EvaluationScore:
+        return cls(
+            name=data.get("name", ""),
+            score=float(data.get("score", 0.0)),
+            unit=data.get("unit", ""),
+            details=data.get("details", ""),
+        )
+
+
+@dataclass
+class EvaluationResult:
+    """Complete evaluation report for an AI execution."""
+    commit_id: str
+    execution_id: str
+    overall_score: float
+    scores: dict[str, EvaluationScore] = field(default_factory=dict)
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "commit_id": self.commit_id,
+            "execution_id": self.execution_id,
+            "overall_score": self.overall_score,
+            "scores": {k: v.to_dict() for k, v in self.scores.items()},
+            "timestamp": self.timestamp,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> EvaluationResult:
+        scores_raw = data.get("scores", {})
+        scores = {}
+        if isinstance(scores_raw, dict):
+            for k, v in scores_raw.items():
+                if isinstance(v, dict):
+                    scores[k] = EvaluationScore.from_dict(v)
+        return cls(
+            id=data.get("id", str(uuid.uuid4())),
+            commit_id=data.get("commit_id", ""),
+            execution_id=data.get("execution_id", ""),
+            overall_score=float(data.get("overall_score", 0.0)),
+            scores=scores,
+            timestamp=data.get("timestamp", datetime.now(timezone.utc).isoformat()),
+        )