runsight-core 0.1.2__py3-none-any.whl

runsight_core/__init__.py

@@ -0,0 +1,44 @@
+"""
+Runsight Agent OS Core Engine
+"""
+
+from .blocks.base import BaseBlock
+from .blocks.code import CodeBlock
+from .blocks.dispatch import DispatchBlock
+from .blocks.gate import GateBlock
+from .blocks.linear import LinearBlock
+from .blocks.loop import CarryContextConfig, LoopBlock, LoopBlockDef
+from .blocks.registry import BlockFactory, BlockRegistry
+from .blocks.synthesize import SynthesizeBlock
+from .blocks.workflow_block import WorkflowBlock
+from .primitives import Soul, Step, Task
+from .runner import ExecutionResult, RunsightTeamRunner
+from .state import BlockResult, WorkflowState
+from .workflow import Workflow
+from .yaml import parse_workflow_yaml
+from .yaml.schema import RetryConfig
+
+__all__ = [
+    "Soul",
+    "Task",
+    "Step",
+    "RunsightTeamRunner",
+    "ExecutionResult",
+    "BlockResult",
+    "WorkflowState",
+    "BaseBlock",
+    "LinearBlock",
+    "DispatchBlock",
+    "SynthesizeBlock",
+    "LoopBlock",
+    "GateBlock",
+    "WorkflowBlock",
+    "CodeBlock",
+    "BlockRegistry",
+    "BlockFactory",
+    "Workflow",
+    "parse_workflow_yaml",
+    "LoopBlockDef",
+    "RetryConfig",
+    "CarryContextConfig",
+]

runsight_core/artifacts.py

@@ -0,0 +1,66 @@
+"""
+ArtifactStore ABC and InMemoryArtifactStore for workflow artifact management.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+
+class ArtifactStore(ABC):
+    """Abstract base class for artifact storage backends."""
+
+    def __init__(self, run_id: str) -> None:
+        self.run_id = run_id
+
+    @abstractmethod
+    async def write(
+        self, key: str, content: str, *, metadata: Optional[Dict[str, Any]] = None
+    ) -> str: ...
+
+    @abstractmethod
+    async def read(self, ref: str) -> str: ...
+
+    @abstractmethod
+    async def list_artifacts(self) -> List[Dict[str, Any]]: ...
+
+    @abstractmethod
+    async def cleanup(self) -> None: ...
+
+
+class InMemoryArtifactStore(ArtifactStore):
+    """In-memory artifact store using mem://{run_id}/{key} refs."""
+
+    def __init__(self, run_id: str) -> None:
+        super().__init__(run_id)
+        self._content: Dict[str, str] = {}
+        self._metadata: Dict[str, Optional[Dict[str, Any]]] = {}
+
+    async def write(
+        self, key: str, content: str, *, metadata: Optional[Dict[str, Any]] = None
+    ) -> str:
+        self._content[key] = content
+        self._metadata[key] = metadata
+        return f"mem://{self.run_id}/{key}"
+
+    async def read(self, ref: str) -> str:
+        prefix = f"mem://{self.run_id}/"
+        if not ref.startswith(prefix):
+            raise KeyError(ref)
+        key = ref[len(prefix) :]
+        if key not in self._content:
+            raise KeyError(ref)
+        return self._content[key]
+
+    async def list_artifacts(self) -> List[Dict[str, Any]]:
+        return [
+            {
+                "key": key,
+                "ref": f"mem://{self.run_id}/{key}",
+                "metadata": self._metadata[key],
+            }
+            for key in self._content
+        ]
+
+    async def cleanup(self) -> None:
+        self._content.clear()
+        self._metadata.clear()

runsight_core/assertions/__init__.py

@@ -0,0 +1,25 @@
+"""Assertion plugin interface for runsight_core."""
+
+from runsight_core.assertions.base import (
+    Assertion,
+    AssertionContext,
+    GradingResult,
+    TokenUsage,
+)
+from runsight_core.assertions.registry import (
+    register_assertion,
+    run_assertion,
+    run_assertions,
+)
+from runsight_core.assertions.scoring import AssertionsResult
+
+__all__ = [
+    "Assertion",
+    "AssertionContext",
+    "AssertionsResult",
+    "GradingResult",
+    "TokenUsage",
+    "register_assertion",
+    "run_assertion",
+    "run_assertions",
+]

runsight_core/assertions/base.py

@@ -0,0 +1,57 @@
+"""Base models for the assertion plugin interface."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Protocol, runtime_checkable
+
+
+@dataclass
+class TokenUsage:
+    """Token usage breakdown for an assertion evaluation."""
+
+    prompt: int = 0
+    completion: int = 0
+    total: int = 0
+
+
+@dataclass
+class GradingResult:
+    """Result of a single assertion evaluation."""
+
+    passed: bool
+    score: float
+    reason: str
+    named_scores: dict[str, float] = field(default_factory=dict)
+    tokens_used: TokenUsage | None = None
+    component_results: list[GradingResult] = field(default_factory=list)
+    assertion_type: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class AssertionContext:
+    """Context provided to assertion evaluators."""
+
+    output: str
+    prompt: str
+    prompt_hash: str
+    soul_id: str
+    soul_version: str
+    block_id: str
+    block_type: str
+    cost_usd: float
+    total_tokens: int
+    latency_ms: float
+    variables: dict[str, Any]
+    run_id: str
+    workflow_id: str
+
+
+@runtime_checkable
+class Assertion(Protocol):
+    """Protocol that assertion plugins must satisfy."""
+
+    type: str
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult: ...

runsight_core/assertions/deterministic/__init__.py

@@ -0,0 +1,65 @@
+"""Deterministic assertion plugins — registers all 15 types on import."""
+
+from runsight_core.assertions.deterministic.linguistic import (
+    BleuAssertion,
+    LevenshteinAssertion,
+    RougeNAssertion,
+)
+from runsight_core.assertions.deterministic.performance import (
+    CostAssertion,
+    LatencyAssertion,
+)
+from runsight_core.assertions.deterministic.string import (
+    ContainsAllAssertion,
+    ContainsAnyAssertion,
+    ContainsAssertion,
+    EqualsAssertion,
+    IContainsAssertion,
+    RegexAssertion,
+    StartsWithAssertion,
+    WordCountAssertion,
+)
+from runsight_core.assertions.deterministic.structural import (
+    ContainsJsonAssertion,
+    IsJsonAssertion,
+)
+from runsight_core.assertions.registry import register_assertion
+
+_ALL_ASSERTIONS: list[type] = [
+    EqualsAssertion,
+    ContainsAssertion,
+    IContainsAssertion,
+    ContainsAllAssertion,
+    ContainsAnyAssertion,
+    StartsWithAssertion,
+    RegexAssertion,
+    WordCountAssertion,
+    IsJsonAssertion,
+    ContainsJsonAssertion,
+    CostAssertion,
+    LatencyAssertion,
+    LevenshteinAssertion,
+    BleuAssertion,
+    RougeNAssertion,
+]
+
+for _cls in _ALL_ASSERTIONS:
+    register_assertion(_cls.type, _cls)
+
+__all__ = [
+    "BleuAssertion",
+    "ContainsAllAssertion",
+    "ContainsAnyAssertion",
+    "ContainsAssertion",
+    "ContainsJsonAssertion",
+    "CostAssertion",
+    "EqualsAssertion",
+    "IContainsAssertion",
+    "IsJsonAssertion",
+    "LatencyAssertion",
+    "LevenshteinAssertion",
+    "RegexAssertion",
+    "RougeNAssertion",
+    "StartsWithAssertion",
+    "WordCountAssertion",
+]

runsight_core/assertions/deterministic/linguistic.py

@@ -0,0 +1,129 @@
+"""Deterministic linguistic assertion plugins.
+
+Covers: levenshtein, bleu, rouge-n.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import Counter
+from typing import Any
+
+import editdistance
+from rouge_score import rouge_scorer
+
+from runsight_core.assertions.base import AssertionContext, GradingResult
+
+
+class LevenshteinAssertion:
+    """Edit distance <= threshold."""
+
+    type = "levenshtein"
+
+    def __init__(self, value: Any = "", threshold: float | None = None) -> None:
+        self.value = str(value)
+        self.threshold = threshold if threshold is not None else 5
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        distance = editdistance.eval(output, self.value)
+        passed = distance <= self.threshold
+        score = 1.0 if passed else 0.0
+        reason = f"Levenshtein distance is {distance} (threshold {self.threshold})"
+        return GradingResult(passed=passed, score=score, reason=reason)
+
+
+class BleuAssertion:
+    """BLEU-4 score >= threshold. Inline implementation (no nltk)."""
+
+    type = "bleu"
+
+    def __init__(self, value: Any = "", threshold: float | None = None) -> None:
+        self.value = str(value)
+        self.threshold = threshold if threshold is not None else 0.5
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        score = _compute_bleu(reference=self.value, candidate=output)
+        passed = score >= self.threshold
+        reason = f"BLEU score {score:.4f} {'>='}  threshold {self.threshold}"
+        if not passed:
+            reason = f"BLEU score {score:.4f} < threshold {self.threshold}"
+        return GradingResult(passed=passed, score=score, reason=reason)
+
+
+class RougeNAssertion:
+    """ROUGE-N score >= threshold using rouge-score library."""
+
+    type = "rouge-n"
+
+    def __init__(self, value: Any = "", threshold: float | None = None) -> None:
+        self.value = str(value)
+        self.threshold = threshold if threshold is not None else 0.75
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        if not output or not self.value:
+            score = 0.0
+        else:
+            scorer = rouge_scorer.RougeScorer(["rouge1"], use_stemmer=False)
+            scores = scorer.score(self.value, output)
+            score = scores["rouge1"].fmeasure
+
+        passed = score >= self.threshold
+        if passed:
+            reason = f"ROUGE-N score {score:.4f} >= threshold {self.threshold}"
+        else:
+            reason = f"ROUGE-N score {score:.4f} < threshold {self.threshold}"
+        return GradingResult(passed=passed, score=score, reason=reason)
+
+
+# ── Inline BLEU-4 implementation ────────────────────────────────────────────
+
+
+def _get_ngrams(tokens: list[str], n: int) -> Counter[tuple[str, ...]]:
+    """Extract n-grams from a token list."""
+    return Counter(tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1))
+
+
+def _compute_bleu(reference: str, candidate: str, max_n: int = 4) -> float:
+    """Compute BLEU score with smoothing (method 1: add 1 to numerator/denominator).
+
+    Ported from promptfoo's BLEU implementation.
+    """
+    ref_tokens = reference.lower().split()
+    cand_tokens = candidate.lower().split()
+
+    if not cand_tokens:
+        return 0.0
+    if not ref_tokens:
+        return 0.0
+
+    # Brevity penalty
+    bp = 1.0
+    if len(cand_tokens) < len(ref_tokens):
+        bp = math.exp(1.0 - len(ref_tokens) / len(cand_tokens))
+
+    # Modified precision for each n-gram order with smoothing
+    log_avg = 0.0
+    for n in range(1, max_n + 1):
+        ref_ngrams = _get_ngrams(ref_tokens, n)
+        cand_ngrams = _get_ngrams(cand_tokens, n)
+
+        # Clipped counts
+        clipped = 0
+        total = 0
+        for ngram, count in cand_ngrams.items():
+            clipped += min(count, ref_ngrams.get(ngram, 0))
+            total += count
+
+        # Smoothing: add 1 to both numerator and denominator when n > 1
+        if n == 1:
+            if total == 0:
+                return 0.0
+            precision = clipped / total
+            if precision == 0:
+                return 0.0
+        else:
+            precision = (clipped + 1) / (total + 1)
+
+        log_avg += math.log(precision) / max_n
+
+    return bp * math.exp(log_avg)

runsight_core/assertions/deterministic/performance.py

@@ -0,0 +1,58 @@
+"""Deterministic performance assertion plugins.
+
+Covers: cost, latency.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from runsight_core.assertions.base import AssertionContext, GradingResult
+
+
+class CostAssertion:
+    """Check that cost_usd from context is within threshold."""
+
+    type = "cost"
+
+    def __init__(self, value: Any = None, threshold: float | None = None) -> None:
+        self.value = value
+        self.threshold = threshold
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        cost = context.cost_usd
+        threshold = self.threshold if self.threshold is not None else 0.0
+        if cost <= threshold:
+            return GradingResult(
+                passed=True,
+                score=1.0,
+                reason=f"Cost ${cost:.4f} is within threshold ${threshold:.4f}",
+            )
+        return GradingResult(
+            passed=False, score=0.0, reason=f"Cost ${cost:.4f} exceeds threshold ${threshold:.4f}"
+        )
+
+
+class LatencyAssertion:
+    """Check that latency_ms from context is within threshold."""
+
+    type = "latency"
+
+    def __init__(self, value: Any = None, threshold: float | None = None) -> None:
+        self.value = value
+        self.threshold = threshold
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        latency = context.latency_ms
+        threshold = self.threshold if self.threshold is not None else 0.0
+        if latency <= threshold:
+            return GradingResult(
+                passed=True,
+                score=1.0,
+                reason=f"Latency {latency:.1f}ms is within threshold {threshold:.1f}ms",
+            )
+        return GradingResult(
+            passed=False,
+            score=0.0,
+            reason=f"Latency {latency:.1f}ms exceeds threshold {threshold:.1f}ms",
+        )

runsight_core/assertions/deterministic/string.py

@@ -0,0 +1,212 @@
+"""Deterministic string assertion plugins.
+
+Covers: equals, contains, icontains, contains-all, contains-any,
+starts-with, regex, word-count.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+
+from runsight_core.assertions.base import AssertionContext, GradingResult
+
+
+class EqualsAssertion:
+    """Exact string match or JSON deep-equal."""
+
+    type = "equals"
+
+    def __init__(self, value: Any = "", threshold: float | None = None) -> None:
+        self.value = value
+        self.threshold = threshold
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        value_str = str(self.value)
+
+        # Try JSON deep-equal first
+        try:
+            output_parsed = json.loads(output)
+            value_parsed = json.loads(value_str)
+            if output_parsed == value_parsed:
+                return GradingResult(
+                    passed=True, score=1.0, reason="Output matches expected value (JSON deep-equal)"
+                )
+            return GradingResult(
+                passed=False,
+                score=0.0,
+                reason=f"JSON values differ: expected {value_str!r}, got {output!r}",
+            )
+        except (json.JSONDecodeError, TypeError):
+            pass
+
+        # Fall back to exact string match
+        if output == value_str:
+            return GradingResult(
+                passed=True, score=1.0, reason="Output exactly matches expected value"
+            )
+        return GradingResult(
+            passed=False, score=0.0, reason=f"Expected {value_str!r}, got {output!r}"
+        )
+
+
+class ContainsAssertion:
+    """Case-sensitive substring check."""
+
+    type = "contains"
+
+    def __init__(self, value: Any = "", threshold: float | None = None) -> None:
+        self.value = str(value)
+        self.threshold = threshold
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        if self.value in output:
+            return GradingResult(passed=True, score=1.0, reason=f"Output contains {self.value!r}")
+        return GradingResult(
+            passed=False, score=0.0, reason=f"Output does not contain {self.value!r}"
+        )
+
+
+class IContainsAssertion:
+    """Case-insensitive substring check."""
+
+    type = "icontains"
+
+    def __init__(self, value: Any = "", threshold: float | None = None) -> None:
+        self.value = str(value)
+        self.threshold = threshold
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        if self.value.lower() in output.lower():
+            return GradingResult(
+                passed=True, score=1.0, reason=f"Output contains {self.value!r} (case-insensitive)"
+            )
+        return GradingResult(
+            passed=False,
+            score=0.0,
+            reason=f"Output does not contain {self.value!r} (case-insensitive)",
+        )
+
+
+class ContainsAllAssertion:
+    """All items in value list must be substrings."""
+
+    type = "contains-all"
+
+    def __init__(self, value: Any = None, threshold: float | None = None) -> None:
+        self.value: list[str] = value if value is not None else []
+        self.threshold = threshold
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        missing = [item for item in self.value if str(item) not in output]
+        if not missing:
+            return GradingResult(
+                passed=True, score=1.0, reason="Output contains all required substrings"
+            )
+        return GradingResult(passed=False, score=0.0, reason=f"Output missing: {missing!r}")
+
+
+class ContainsAnyAssertion:
+    """At least one item in value list must be a substring."""
+
+    type = "contains-any"
+
+    def __init__(self, value: Any = None, threshold: float | None = None) -> None:
+        self.value: list[str] = value if value is not None else []
+        self.threshold = threshold
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        if not self.value:
+            return GradingResult(passed=False, score=0.0, reason="No candidates provided to match")
+        for item in self.value:
+            if str(item) in output:
+                return GradingResult(passed=True, score=1.0, reason=f"Output contains {item!r}")
+        return GradingResult(
+            passed=False, score=0.0, reason=f"Output does not contain any of {self.value!r}"
+        )
+
+
+class StartsWithAssertion:
+    """String prefix check."""
+
+    type = "starts-with"
+
+    def __init__(self, value: Any = "", threshold: float | None = None) -> None:
+        self.value = str(value)
+        self.threshold = threshold
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        if output.startswith(self.value):
+            return GradingResult(
+                passed=True, score=1.0, reason=f"Output starts with {self.value!r}"
+            )
+        return GradingResult(
+            passed=False, score=0.0, reason=f"Output does not start with {self.value!r}"
+        )
+
+
+class RegexAssertion:
+    """Regex search match."""
+
+    type = "regex"
+
+    def __init__(self, value: Any = "", threshold: float | None = None) -> None:
+        self.value = str(value)
+        self.threshold = threshold
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        try:
+            if re.search(self.value, output):
+                return GradingResult(
+                    passed=True, score=1.0, reason=f"Output matches pattern {self.value!r}"
+                )
+            return GradingResult(
+                passed=False, score=0.0, reason=f"Output does not match pattern {self.value!r}"
+            )
+        except re.error as e:
+            return GradingResult(passed=False, score=0.0, reason=f"Invalid regex pattern: {e}")
+
+
+class WordCountAssertion:
+    """Word count check: exact int or {min, max} range."""
+
+    type = "word-count"
+
+    def __init__(self, value: Any = None, threshold: float | None = None) -> None:
+        self.value = value
+        self.threshold = threshold
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        words = output.split()
+        count = len(words)
+
+        if isinstance(self.value, int):
+            if count == self.value:
+                return GradingResult(
+                    passed=True, score=1.0, reason=f"Word count is {count} (expected {self.value})"
+                )
+            return GradingResult(
+                passed=False, score=0.0, reason=f"Word count is {count}, expected {self.value}"
+            )
+
+        if isinstance(self.value, dict):
+            min_val = self.value.get("min")
+            max_val = self.value.get("max")
+
+            if min_val is not None and max_val is not None and min_val > max_val:
+                raise ValueError(f"min ({min_val}) is greater than max ({max_val})")
+
+            if min_val is not None and count < min_val:
+                return GradingResult(
+                    passed=False, score=0.0, reason=f"Word count {count} is below minimum {min_val}"
+                )
+            if max_val is not None and count > max_val:
+                return GradingResult(
+                    passed=False, score=0.0, reason=f"Word count {count} exceeds maximum {max_val}"
+                )
+            return GradingResult(
+                passed=True, score=1.0, reason=f"Word count {count} is within range"
+            )
+
+        raise TypeError(f"Invalid value type for word-count: {type(self.value).__name__}")