dslighting 1.3.9__py3-none-any.whl

dsat/services/states/experience.py

@@ -0,0 +1,237 @@
+"""
+Implements Experience, which manages the state of a meta-optimization process.
+This is the core state representation for Paradigm 3 (AFlow-style) evolutionary search.
+"""
+import json
+import logging
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional, List, Any, Dict, Tuple
+
+import numpy as np
+
+from dsat.models.candidates import WorkflowCandidate
+from dsat.services.states.base import State
+from dsat.services.workspace import WorkspaceService
+
+logger = logging.getLogger(__name__)
+
+class Experience(State):
+    """
+    Acts as the database for the meta-optimizer. It saves and loads
+    workflow scores and modification history to guide the search process,
+    persisting state to the filesystem within the run's workspace.
+    """
+    def __init__(self, workspace: WorkspaceService):
+        self.workspace = workspace
+        # Define paths within the managed workspace
+        self.scores_file = workspace.get_path("state") / "scores.jsonl"
+        self.experience_file = workspace.get_path("state") / "experience.json"
+        self.candidates_dir = workspace.get_path("candidates")
+        
+        # Initialize state files if they don't exist
+        self.scores_file.touch()
+        if not self.experience_file.exists():
+            with open(self.experience_file, 'w') as f:
+                json.dump({}, f)
+
+    def _load_all_candidates(self) -> List[WorkflowCandidate]:
+        """Loads all recorded candidates from the scores file."""
+        candidates = []
+        if not self.scores_file.exists() or self.scores_file.stat().st_size == 0:
+            return []
+            
+        with open(self.scores_file, "r", encoding="utf-8") as f:
+            for line in f:
+                try:
+                    data = json.loads(line)
+                    score_type = data.get("score_type", "fitness")
+                    if score_type not in {"fitness", "fine"}:
+                        continue
+                    code_path = Path(data['code_path'])
+                    if code_path.exists():
+                        with open(code_path, "r", encoding="utf-8") as code_file:
+                            code = code_file.read()
+                        candidates.append(
+                            WorkflowCandidate(
+                                workflow_code=code, 
+                                fitness=data['fitness'],
+                                round_num=data.get('round')
+                            )
+                        )
+                except (json.JSONDecodeError, KeyError) as e:
+                    logger.warning(f"Skipping malformed line in scores.jsonl: {e}")
+        return candidates
+
+    def get_experience_summary(self, parent_round_num: Optional[int]) -> str:
+        """
+        Loads and formats the experience log for a specific parent candidate.
+        """
+        if parent_round_num is None:
+            parent_round_num = -1
+
+        if not self.experience_file.exists():
+            return "Experience log not found."
+
+        with open(self.experience_file, "r", encoding="utf-8") as f:
+            try:
+                all_experience = json.load(f)
+            except json.JSONDecodeError:
+                return "Could not parse experience log."
+
+        if not isinstance(all_experience, dict):
+            return "Could not parse experience log."
+
+        def _coerce_list(value: Any) -> list[dict]:
+            if not isinstance(value, list):
+                return []
+            out: list[dict] = []
+            for item in value:
+                if isinstance(item, dict):
+                    out.append(item)
+            return out
+
+        summary_lines = []
+        if parent_round_num >= 0:
+            summary_lines.append(f"History of modifications for parent from round {parent_round_num}:")
+        else:
+            summary_lines.append("History of modifications (no specific parent selected):")
+
+        parent_key = str(parent_round_num)
+        parent_exp = all_experience.get(parent_key) if isinstance(all_experience.get(parent_key), dict) else {}
+        parent_success = _coerce_list(parent_exp.get("success"))
+        parent_failure = _coerce_list(parent_exp.get("failure"))
+
+        summary_lines.append("\n### Successful Modifications:")
+        if parent_success:
+            for mod in parent_success:
+                child = mod.get("child_round")
+                score_after = float(mod.get("score_after", 0.0) or 0.0)
+                delta = mod.get("delta")
+                delta_str = ""
+                if isinstance(delta, (int, float)):
+                    delta_str = f", Δ={float(delta):+.4f}"
+                summary_lines.append(
+                    f"- (Child Round {child}, New Score: {score_after:.4f}{delta_str}) {mod.get('modification','')}"
+                )
+        else:
+            summary_lines.append("- (none yet)")
+
+        summary_lines.append("\n### Failed Modifications:")
+        if parent_failure:
+            for mod in parent_failure:
+                child = mod.get("child_round")
+                score_after = float(mod.get("score_after", 0.0) or 0.0)
+                delta = mod.get("delta")
+                delta_str = ""
+                if isinstance(delta, (int, float)):
+                    delta_str = f", Δ={float(delta):+.4f}"
+                summary_lines.append(
+                    f"- (Child Round {child}, New Score: {score_after:.4f}{delta_str}) {mod.get('modification','')}"
+                )
+        else:
+            summary_lines.append("- (none yet)")
+
+        # Add global successful examples to give the optimizer concrete positive patterns,
+        # even when the selected parent has no successes yet.
+        global_success: list[Tuple[float, str, dict]] = []
+        for pkey, pexp in all_experience.items():
+            if not isinstance(pexp, dict):
+                continue
+            for mod in _coerce_list(pexp.get("success")):
+                try:
+                    score_after = float(mod.get("score_after", 0.0) or 0.0)
+                except Exception:
+                    score_after = 0.0
+                global_success.append((score_after, str(pkey), mod))
+
+        global_success.sort(key=lambda t: t[0], reverse=True)
+        top_global = global_success[:5]
+        summary_lines.append("\n### Successful Examples (Global Top-5):")
+        if top_global:
+            for score_after, pkey, mod in top_global:
+                child = mod.get("child_round")
+                delta = mod.get("delta")
+                delta_str = ""
+                if isinstance(delta, (int, float)):
+                    delta_str = f", Δ={float(delta):+.4f}"
+                summary_lines.append(
+                    f"- (Parent {pkey} → Child {child}, Score: {score_after:.4f}{delta_str}) {mod.get('modification','')}"
+                )
+        else:
+            summary_lines.append("- (none yet)")
+        
+        return "\n".join(summary_lines)
+
+    def select_parent_candidate(self, top_k: int) -> Optional[WorkflowCandidate]:
+        """
+        Selects a parent candidate using a softmax probability distribution over the
+        top_k best-performing unique candidates, balancing exploration and exploitation.
+        """
+        all_candidates = self._load_all_candidates()
+        if not all_candidates:
+            return None
+
+        # Sort by fitness (higher is better) and take the top k
+        sorted_candidates = sorted(all_candidates, key=lambda c: c.fitness or -1.0, reverse=True)
+        top_candidates = sorted_candidates[:top_k]
+
+        if not top_candidates:
+            return None
+        
+        fitness_scores = np.array([c.fitness for c in top_candidates])
+        # Softmax probabilities: e^score / sum(e^scores)
+        probabilities = np.exp(fitness_scores) / np.sum(np.exp(fitness_scores))
+
+        return np.random.choice(top_candidates, p=probabilities)
+
+    def record_score(
+        self,
+        round_num: int,
+        fitness: float,
+        code: str,
+        *,
+        score_type: str = "fitness",
+        extra: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """Saves the workflow code and appends its score to the log."""
+        candidate_code_path = self.candidates_dir / f"round_{round_num}_workflow.py"
+        with open(candidate_code_path, "w", encoding="utf-8") as f:
+            f.write(code)
+
+        with open(self.scores_file, "a", encoding="utf-8") as f:
+            payload: Dict[str, Any] = {
+                "round": round_num,
+                "fitness": fitness,
+                "code_path": str(candidate_code_path),
+                "score_type": str(score_type or "fitness"),
+            }
+            if extra:
+                payload["extra"] = extra
+            f.write(json.dumps(payload) + "\n")
+
+    def record_experience(self, parent_round: int, child_round: int, modification: str, score_before: float, score_after: float):
+        """Records the outcome of a modification attempt in the experience log."""
+        with open(self.experience_file, 'r+') as f:
+            data = json.load(f)
+            parent_key = str(parent_round)
+            if parent_key not in data:
+                data[parent_key] = {"success": [], "failure": []}
+            
+            outcome = {
+                "child_round": child_round,
+                "modification": modification,
+                "score_before": float(score_before),
+                "score_after": score_after,
+                "delta": float(score_after) - float(score_before),
+                "recorded_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
+            }
+            
+            if score_after > score_before:
+                data[parent_key]["success"].append(outcome)
+            else:
+                data[parent_key]["failure"].append(outcome)
+            
+            f.seek(0)
+            json.dump(data, f, indent=4)
+            f.truncate()

dsat/services/states/journal.py

@@ -0,0 +1,153 @@
+# dsat/services/states/journal.py
+
+"""
+Implements JournalState, which manages a tree of solution attempts (Nodes).
+This is the core state representation for Paradigm 2 (AIDE/AutoMind-style) search agents.
+"""
+import uuid
+from functools import total_ordering
+from typing import Optional, Any, List, Dict, Set
+
+from pydantic import BaseModel, Field
+
+from dsat.common.typing import ExecutionResult
+from dsat.utils.context import truncate_output
+from dsat.services.states.base import State
+
+
+@total_ordering
+class MetricValue(BaseModel):
+    """
+    Represents a comparable metric that can be configured for maximization or minimization.
+    A value of None is considered worse than any numeric value.
+    """
+    value: Optional[float]
+    maximize: bool = True
+
+    def __gt__(self, other: "MetricValue") -> bool:
+        if self.value is None:
+            return False
+        if other.value is None:
+            return True
+        return (self.value > other.value) if self.maximize else (self.value < other.value)
+
+    def __eq__(self, other: Any) -> bool:
+        return isinstance(other, MetricValue) and self.value == other.value
+
+    def __str__(self) -> str:
+        direction = "↑" if self.maximize else "↓"
+        val_str = f"{self.value:.4f}" if self.value is not None else "N/A"
+        return f"Metric{direction}({val_str})"
+
+class Node(BaseModel):
+    """
+    Represents a single attempt or node in the solution search tree.
+    Each node contains the code, plan, execution results, and review analysis.
+    """
+    code: str
+    plan: str
+
+    id: str = Field(default_factory=lambda: uuid.uuid4().hex)
+    parent_id: Optional[str] = None
+    children_ids: Set[str] = Field(default_factory=set)
+
+    # Execution Results
+    term_out: str = ""
+    exec_time: float = 0.0
+    exc_type: Optional[str] = None
+    exec_metadata: Dict[str, Any] = Field(default_factory=dict)
+
+    # LLM Recordings
+    task_context: Dict[str, Any] = Field(default_factory=dict)
+    generate_prompt: Optional[str] = None
+    llm_generate: Optional[Dict[str, Any]] = None
+    review_context: Optional[Dict[str, Any]] = None
+    llm_review: Optional[Dict[str, Any]] = None
+
+    # Review Results
+    analysis: str = ""
+    metric: MetricValue = Field(default_factory=lambda: MetricValue(value=None))
+    is_buggy: bool = True
+    step: int = -1
+
+    # Artifact paths
+    code_artifact_path: Optional[str] = None
+    final_submission_path: Optional[str] = None
+
+    def absorb_exec_result(self, exec_result: ExecutionResult):
+        """Updates the node with the results from a sandbox execution."""
+        stdout = exec_result.stdout or ""
+        stderr = exec_result.stderr or ""
+        combined_output = f"STDOUT:\n{stdout}\n\nSTDERR:\n{stderr}".strip()
+        self.term_out = truncate_output(combined_output)
+        self.exc_type = exec_result.exc_type
+        self.is_buggy = not exec_result.success
+        self.exec_metadata = exec_result.metadata or {}
+
+    class Config:
+        """Pydantic configuration."""
+        json_encoders = {set: list} # Allow sets to be serialized to lists in JSON
+
+class JournalState(State, BaseModel):
+    """
+    Manages the entire search tree (the "Journal") of solution nodes.
+    Provides methods for appending nodes, traversing the tree, and selecting
+    nodes based on different criteria (e.g., best, buggy).
+    """
+    nodes: Dict[str, Node] = Field(default_factory=dict)
+
+    def __len__(self) -> int:
+        return len(self.nodes)
+
+    def append(self, node: Node, parent: Optional[Node] = None):
+        """Adds a new node to the journal, linking it to a parent if provided."""
+        if parent:
+            if parent.id not in self.nodes:
+                 raise ValueError(f"Parent node with id {parent.id} not in journal.")
+            node.parent_id = parent.id
+            self.nodes[parent.id].children_ids.add(node.id)
+        node.step = len(self)
+        self.nodes[node.id] = node
+
+    def get_node(self, node_id: str) -> Optional[Node]:
+        """Retrieves a node by its ID."""
+        return self.nodes.get(node_id)
+
+    def get_best_node(self) -> Optional[Node]:
+        """Finds the best-performing, non-buggy node in the entire journal."""
+        good_nodes = [n for n in self.nodes.values() if not n.is_buggy]
+        if not good_nodes:
+            return None
+        return max(good_nodes, key=lambda n: n.metric)
+
+    def generate_summary(self, max_nodes: int = 3) -> str:
+        """
+        Creates a textual summary of successful past attempts for prompt context.
+        MODIFIED: Now selects the `max_nodes` BEST performing successful attempts.
+        """
+        good_nodes = sorted(
+            [n for n in self.nodes.values() if not n.is_buggy and n.metric.value is not None],
+            key=lambda x: x.metric,
+            reverse=True  # MetricValue handles > comparison correctly, so reverse=True gets the best
+        )
+        if not good_nodes:
+            return "No successful solutions have been found yet."
+
+        # Apply windowing: take the top `max_nodes`
+        selected_nodes = good_nodes[:max_nodes]
+
+        summary_parts = []
+        for n in selected_nodes:
+            summary_part = (
+                f"Attempt #{n.step}:\n"
+                f"Plan: {n.plan}\n"
+                f"Result Analysis: {n.analysis}\n"
+                f"Validation Metric: {n.metric}\n"
+            )
+            summary_parts.append(summary_part)
+        
+        prefix = ""
+        if len(good_nodes) > len(selected_nodes):
+            prefix = f"[... {len(good_nodes) - len(selected_nodes)} other successful attempts exist ...]\n"
+
+        return prefix + "Here is a summary of the best performing attempts:\n" + "\n------------------\n".join(summary_parts)

dsat/services/states/operator_library.py

@@ -0,0 +1,290 @@
+"""
+Persistent library for discovered operators.
+
+This stores LLM-proposed Operator code + metadata across runs, and tracks basic
+usage/success statistics to support simple operator-level selection.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional, Tuple
+
+
+def _utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def _safe_int(value: Any, default: int = 0) -> int:
+    try:
+        return int(value)
+    except Exception:
+        return default
+
+
+@dataclass(frozen=True)
+class LibraryOperatorSpec:
+    name: str
+    version: int
+    code: str
+    description: str
+    inputs: str
+    outputs: str
+    triggers: str
+    task_types: List[str]
+    uses: int
+    successes: int
+    failures: int
+
+
+class OperatorLibrary:
+    def __init__(self, path: Path):
+        self.path = path
+        self._data: Dict[str, Any] = {"operators": {}}
+        self._load()
+
+    def _load(self) -> None:
+        if not self.path.exists():
+            self._data = {"operators": {}}
+            return
+        try:
+            payload = json.loads(self.path.read_text(encoding="utf-8"))
+        except Exception:
+            self._data = {"operators": {}}
+            return
+
+        if not isinstance(payload, dict) or not isinstance(payload.get("operators"), dict):
+            self._data = {"operators": {}}
+            return
+        self._data = payload
+
+    def _save(self) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = self.path.with_suffix(self.path.suffix + ".tmp")
+        tmp.write_text(
+            json.dumps(self._data, ensure_ascii=False, indent=2, sort_keys=True),
+            encoding="utf-8",
+        )
+        tmp.replace(self.path)
+
+    def has(self, name: str) -> bool:
+        return name in self._data.get("operators", {})
+
+    def get_best_version(self, name: str) -> Optional[LibraryOperatorSpec]:
+        """
+        Return the best available version for an operator name.
+
+        Selection is based on (success_rate, uses, version) in descending order.
+        Returns None if the operator is missing or has no usable versions.
+        """
+        operators = self._data.get("operators", {})
+        if not isinstance(operators, dict):
+            return None
+
+        record = operators.get(name)
+        if not isinstance(record, dict):
+            return None
+        versions = record.get("versions", [])
+        if not isinstance(versions, list) or not versions:
+            return None
+
+        best: Optional[Dict[str, Any]] = None
+        best_key: Optional[tuple[float, int, int]] = None
+        best_version = 1
+
+        for v in versions:
+            if not isinstance(v, dict):
+                continue
+            code = str(v.get("code") or "")
+            if not code.strip():
+                continue
+            uses = _safe_int(v.get("uses"), 0)
+            successes = _safe_int(v.get("successes"), 0)
+            version = _safe_int(v.get("version"), 1)
+            success_rate = successes / uses if uses > 0 else 0.0
+            key = (float(success_rate), int(uses), int(version))
+            if best_key is None or key > best_key:
+                best = v
+                best_key = key
+                best_version = version
+
+        if best is None:
+            return None
+
+        return LibraryOperatorSpec(
+            name=str(name),
+            version=int(best_version),
+            code=str(best.get("code") or ""),
+            description=str(best.get("description") or "").strip(),
+            inputs=str(best.get("inputs") or "").strip(),
+            outputs=str(best.get("outputs") or "").strip(),
+            triggers=str(best.get("triggers") or "").strip(),
+            task_types=list(best.get("task_types") or []),
+            uses=_safe_int(best.get("uses"), 0),
+            successes=_safe_int(best.get("successes"), 0),
+            failures=_safe_int(best.get("failures"), 0),
+        )
+
+    def add_version(
+        self,
+        name: str,
+        *,
+        code: str,
+        description: str,
+        inputs: str = "",
+        outputs: str = "",
+        triggers: str = "",
+        task_types: Optional[Iterable[str]] = None,
+    ) -> int:
+        operators = self._data.setdefault("operators", {})
+        record = operators.setdefault(name, {"versions": []})
+        versions: List[Dict[str, Any]] = record.setdefault("versions", [])
+
+        code_hash = hashlib.sha256(code.encode("utf-8")).hexdigest()[:16]
+        for v in versions:
+            if v.get("code_hash") == code_hash:
+                return _safe_int(v.get("version"), default=1)
+
+        next_version = 1 + max((_safe_int(v.get("version"), 0) for v in versions), default=0)
+        versions.append(
+            {
+                "version": next_version,
+                "code_hash": code_hash,
+                "code": code,
+                "description": (description or "").strip(),
+                "inputs": (inputs or "").strip(),
+                "outputs": (outputs or "").strip(),
+                "triggers": (triggers or "").strip(),
+                "task_types": [t for t in (task_types or []) if isinstance(t, str) and t.strip()],
+                "created_at": _utc_now_iso(),
+                "uses": 0,
+                "successes": 0,
+                "failures": 0,
+                "seen_competition_ids": [],
+            }
+        )
+        self._save()
+        return next_version
+
+    def record_outcome(
+        self,
+        name: str,
+        version: int,
+        *,
+        success: bool,
+        competition_ids: Optional[Iterable[str]] = None,
+    ) -> None:
+        operators = self._data.get("operators", {})
+        record = operators.get(name)
+        if not isinstance(record, dict):
+            return
+        versions = record.get("versions", [])
+        if not isinstance(versions, list):
+            return
+
+        for v in versions:
+            if _safe_int(v.get("version"), -1) != int(version):
+                continue
+            v["uses"] = _safe_int(v.get("uses"), 0) + 1
+            if success:
+                v["successes"] = _safe_int(v.get("successes"), 0) + 1
+                v["last_success_at"] = _utc_now_iso()
+            else:
+                v["failures"] = _safe_int(v.get("failures"), 0) + 1
+            v["last_used_at"] = _utc_now_iso()
+
+            if competition_ids:
+                seen = set(v.get("seen_competition_ids") or [])
+                for cid in competition_ids:
+                    if isinstance(cid, str) and cid:
+                        seen.add(cid)
+                v["seen_competition_ids"] = sorted(seen)
+
+            self._save()
+            return
+
+    def select_for_prompt(
+        self,
+        max_ops: int,
+        *,
+        competition_ids: Optional[Iterable[str]] = None,
+        task_types: Optional[Iterable[str]] = None,
+    ) -> List[LibraryOperatorSpec]:
+        """
+        Select up to `max_ops` operators (best version per operator) using a simple UCB score.
+        Adds a small bonus when an operator has succeeded on the same competition_ids or task_types.
+        """
+        operators = self._data.get("operators", {})
+        if not isinstance(operators, dict) or max_ops <= 0:
+            return []
+
+        preferred_competitions = {
+            cid for cid in (competition_ids or []) if isinstance(cid, str) and cid.strip()
+        }
+        preferred_task_types = {
+            t.strip().lower() for t in (task_types or []) if isinstance(t, str) and t.strip()
+        }
+
+        all_versions: List[Tuple[str, Dict[str, Any]]] = []
+        for name, record in operators.items():
+            if not isinstance(record, dict):
+                continue
+            for v in record.get("versions", []) or []:
+                if isinstance(v, dict):
+                    all_versions.append((name, v))
+
+        total_uses = sum(_safe_int(v.get("uses"), 0) for _, v in all_versions)
+        total_uses = max(1, total_uses)
+
+        best_by_name: Dict[str, Tuple[float, Dict[str, Any]]] = {}
+        for name, v in all_versions:
+            uses = _safe_int(v.get("uses"), 0)
+            successes = _safe_int(v.get("successes"), 0)
+            mean = successes / max(1, uses)
+            ucb = mean + math.sqrt(2.0 * math.log(total_uses + 1.0) / (uses + 1.0))
+
+            bonus = 0.0
+            if preferred_competitions:
+                seen = {c for c in (v.get("seen_competition_ids") or []) if isinstance(c, str) and c}
+                if seen & preferred_competitions:
+                    bonus += 0.25
+            if preferred_task_types:
+                vtypes = {
+                    t.strip().lower()
+                    for t in (v.get("task_types") or [])
+                    if isinstance(t, str) and t.strip()
+                }
+                if vtypes & preferred_task_types:
+                    bonus += 0.15
+
+            score = ucb + bonus
+            prev = best_by_name.get(name)
+            if prev is None or score > prev[0]:
+                best_by_name[name] = (score, v)
+
+        ranked = sorted(best_by_name.items(), key=lambda kv: kv[1][0], reverse=True)
+        selected = ranked[:max_ops]
+
+        specs: List[LibraryOperatorSpec] = []
+        for name, (_score, v) in selected:
+            specs.append(
+                LibraryOperatorSpec(
+                    name=name,
+                    version=_safe_int(v.get("version"), 1),
+                    code=str(v.get("code") or ""),
+                    description=str(v.get("description") or "").strip(),
+                    inputs=str(v.get("inputs") or "").strip(),
+                    outputs=str(v.get("outputs") or "").strip(),
+                    triggers=str(v.get("triggers") or "").strip(),
+                    task_types=list(v.get("task_types") or []),
+                    uses=_safe_int(v.get("uses"), 0),
+                    successes=_safe_int(v.get("successes"), 0),
+                    failures=_safe_int(v.get("failures"), 0),
+                )
+            )
+        return specs