runsight-core 1.1.0__tar.gz

runsight_core-1.1.0/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 2.4
+Name: runsight-core
+Version: 1.1.0
+Summary: Runsight Agent OS Core Engine
+License-Expression: Apache-2.0
+Requires-Python: >=3.11
+Requires-Dist: pydantic>=2.0
+Requires-Dist: litellm>=1.0.0
+Requires-Dist: openai>=1.0.0
+Requires-Dist: httpx>=0.27
+Requires-Dist: jsonschema>=4.0
+Requires-Dist: editdistance>=0.6
+Requires-Dist: rouge-score>=0.1
+Requires-Dist: jsonpath-ng>=1.6
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
+Requires-Dist: httpx>=0.27; extra == "dev"
+Requires-Dist: respx>=0.22; extra == "dev"

runsight_core-1.1.0/README.md

@@ -0,0 +1,6 @@
+# packages/core
+
+Canonical home for the reusable Runsight runtime engine.
+
+This package owns the Python runtime engine, its schema snapshot, and the core
+test suite.

runsight_core-1.1.0/pyproject.toml

@@ -0,0 +1,45 @@
+[build-system]
+requires = ["setuptools>=75.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "runsight-core"
+version = "1.1.0"
+description = "Runsight Agent OS Core Engine"
+requires-python = ">=3.11"
+license = "Apache-2.0"
+dependencies = [
+    "pydantic>=2.0",
+    "litellm>=1.0.0",
+    "openai>=1.0.0",
+    "httpx>=0.27",
+    "jsonschema>=4.0",
+    "editdistance>=0.6",
+    "rouge-score>=0.1",
+    "jsonpath-ng>=1.6",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest-asyncio>=0.23",
+    "httpx>=0.27",
+    "respx>=0.22",
+]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["runsight_core*"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+
+[tool.mypy]
+python_version = "3.11"
+strict = true
+warn_unused_configs = true
+disallow_untyped_defs = true
+disallow_any_unimported = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_no_return = true

runsight_core-1.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

runsight_core-1.1.0/src/runsight_core/__init__.py

@@ -0,0 +1,44 @@
+"""
+Runsight Agent OS Core Engine
+"""
+
+from .blocks.base import BaseBlock
+from .blocks.code import CodeBlock
+from .blocks.dispatch import DispatchBlock
+from .blocks.gate import GateBlock
+from .blocks.linear import LinearBlock
+from .blocks.loop import CarryContextConfig, LoopBlock, LoopBlockDef
+from .blocks.registry import BlockFactory, BlockRegistry
+from .blocks.synthesize import SynthesizeBlock
+from .blocks.workflow_block import WorkflowBlock
+from .primitives import Soul, Step, Task
+from .runner import ExecutionResult, RunsightTeamRunner
+from .state import BlockResult, WorkflowState
+from .workflow import Workflow
+from .yaml import parse_workflow_yaml
+from .yaml.schema import RetryConfig
+
+__all__ = [
+    "Soul",
+    "Task",
+    "Step",
+    "RunsightTeamRunner",
+    "ExecutionResult",
+    "BlockResult",
+    "WorkflowState",
+    "BaseBlock",
+    "LinearBlock",
+    "DispatchBlock",
+    "SynthesizeBlock",
+    "LoopBlock",
+    "GateBlock",
+    "WorkflowBlock",
+    "CodeBlock",
+    "BlockRegistry",
+    "BlockFactory",
+    "Workflow",
+    "parse_workflow_yaml",
+    "LoopBlockDef",
+    "RetryConfig",
+    "CarryContextConfig",
+]

runsight_core-1.1.0/src/runsight_core/artifacts.py

@@ -0,0 +1,66 @@
+"""
+ArtifactStore ABC and InMemoryArtifactStore for workflow artifact management.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+
+class ArtifactStore(ABC):
+    """Abstract base class for artifact storage backends."""
+
+    def __init__(self, run_id: str) -> None:
+        self.run_id = run_id
+
+    @abstractmethod
+    async def write(
+        self, key: str, content: str, *, metadata: Optional[Dict[str, Any]] = None
+    ) -> str: ...
+
+    @abstractmethod
+    async def read(self, ref: str) -> str: ...
+
+    @abstractmethod
+    async def list_artifacts(self) -> List[Dict[str, Any]]: ...
+
+    @abstractmethod
+    async def cleanup(self) -> None: ...
+
+
+class InMemoryArtifactStore(ArtifactStore):
+    """In-memory artifact store using mem://{run_id}/{key} refs."""
+
+    def __init__(self, run_id: str) -> None:
+        super().__init__(run_id)
+        self._content: Dict[str, str] = {}
+        self._metadata: Dict[str, Optional[Dict[str, Any]]] = {}
+
+    async def write(
+        self, key: str, content: str, *, metadata: Optional[Dict[str, Any]] = None
+    ) -> str:
+        self._content[key] = content
+        self._metadata[key] = metadata
+        return f"mem://{self.run_id}/{key}"
+
+    async def read(self, ref: str) -> str:
+        prefix = f"mem://{self.run_id}/"
+        if not ref.startswith(prefix):
+            raise KeyError(ref)
+        key = ref[len(prefix) :]
+        if key not in self._content:
+            raise KeyError(ref)
+        return self._content[key]
+
+    async def list_artifacts(self) -> List[Dict[str, Any]]:
+        return [
+            {
+                "key": key,
+                "ref": f"mem://{self.run_id}/{key}",
+                "metadata": self._metadata[key],
+            }
+            for key in self._content
+        ]
+
+    async def cleanup(self) -> None:
+        self._content.clear()
+        self._metadata.clear()

runsight_core-1.1.0/src/runsight_core/assertions/__init__.py

@@ -0,0 +1,25 @@
+"""Assertion plugin interface for runsight_core."""
+
+from runsight_core.assertions.base import (
+    Assertion,
+    AssertionContext,
+    GradingResult,
+    TokenUsage,
+)
+from runsight_core.assertions.registry import (
+    register_assertion,
+    run_assertion,
+    run_assertions,
+)
+from runsight_core.assertions.scoring import AssertionsResult
+
+__all__ = [
+    "Assertion",
+    "AssertionContext",
+    "AssertionsResult",
+    "GradingResult",
+    "TokenUsage",
+    "register_assertion",
+    "run_assertion",
+    "run_assertions",
+]

runsight_core-1.1.0/src/runsight_core/assertions/base.py

@@ -0,0 +1,57 @@
+"""Base models for the assertion plugin interface."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Protocol, runtime_checkable
+
+
+@dataclass
+class TokenUsage:
+    """Token usage breakdown for an assertion evaluation."""
+
+    prompt: int = 0
+    completion: int = 0
+    total: int = 0
+
+
+@dataclass
+class GradingResult:
+    """Result of a single assertion evaluation."""
+
+    passed: bool
+    score: float
+    reason: str
+    named_scores: dict[str, float] = field(default_factory=dict)
+    tokens_used: TokenUsage | None = None
+    component_results: list[GradingResult] = field(default_factory=list)
+    assertion_type: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class AssertionContext:
+    """Context provided to assertion evaluators."""
+
+    output: str
+    prompt: str
+    prompt_hash: str
+    soul_id: str
+    soul_version: str
+    block_id: str
+    block_type: str
+    cost_usd: float
+    total_tokens: int
+    latency_ms: float
+    variables: dict[str, Any]
+    run_id: str
+    workflow_id: str
+
+
+@runtime_checkable
+class Assertion(Protocol):
+    """Protocol that assertion plugins must satisfy."""
+
+    type: str
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult: ...

runsight_core-1.1.0/src/runsight_core/assertions/deterministic/__init__.py

@@ -0,0 +1,65 @@
+"""Deterministic assertion plugins — registers all 15 types on import."""
+
+from runsight_core.assertions.deterministic.linguistic import (
+    BleuAssertion,
+    LevenshteinAssertion,
+    RougeNAssertion,
+)
+from runsight_core.assertions.deterministic.performance import (
+    CostAssertion,
+    LatencyAssertion,
+)
+from runsight_core.assertions.deterministic.string import (
+    ContainsAllAssertion,
+    ContainsAnyAssertion,
+    ContainsAssertion,
+    EqualsAssertion,
+    IContainsAssertion,
+    RegexAssertion,
+    StartsWithAssertion,
+    WordCountAssertion,
+)
+from runsight_core.assertions.deterministic.structural import (
+    ContainsJsonAssertion,
+    IsJsonAssertion,
+)
+from runsight_core.assertions.registry import register_assertion
+
+_ALL_ASSERTIONS: list[type] = [
+    EqualsAssertion,
+    ContainsAssertion,
+    IContainsAssertion,
+    ContainsAllAssertion,
+    ContainsAnyAssertion,
+    StartsWithAssertion,
+    RegexAssertion,
+    WordCountAssertion,
+    IsJsonAssertion,
+    ContainsJsonAssertion,
+    CostAssertion,
+    LatencyAssertion,
+    LevenshteinAssertion,
+    BleuAssertion,
+    RougeNAssertion,
+]
+
+for _cls in _ALL_ASSERTIONS:
+    register_assertion(_cls.type, _cls)
+
+__all__ = [
+    "BleuAssertion",
+    "ContainsAllAssertion",
+    "ContainsAnyAssertion",
+    "ContainsAssertion",
+    "ContainsJsonAssertion",
+    "CostAssertion",
+    "EqualsAssertion",
+    "IContainsAssertion",
+    "IsJsonAssertion",
+    "LatencyAssertion",
+    "LevenshteinAssertion",
+    "RegexAssertion",
+    "RougeNAssertion",
+    "StartsWithAssertion",
+    "WordCountAssertion",
+]

runsight_core-1.1.0/src/runsight_core/assertions/deterministic/linguistic.py

@@ -0,0 +1,129 @@
+"""Deterministic linguistic assertion plugins.
+
+Covers: levenshtein, bleu, rouge-n.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import Counter
+from typing import Any
+
+import editdistance
+from rouge_score import rouge_scorer
+
+from runsight_core.assertions.base import AssertionContext, GradingResult
+
+
+class LevenshteinAssertion:
+    """Edit distance <= threshold."""
+
+    type = "levenshtein"
+
+    def __init__(self, value: Any = "", threshold: float | None = None) -> None:
+        self.value = str(value)
+        self.threshold = threshold if threshold is not None else 5
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        distance = editdistance.eval(output, self.value)
+        passed = distance <= self.threshold
+        score = 1.0 if passed else 0.0
+        reason = f"Levenshtein distance is {distance} (threshold {self.threshold})"
+        return GradingResult(passed=passed, score=score, reason=reason)
+
+
+class BleuAssertion:
+    """BLEU-4 score >= threshold. Inline implementation (no nltk)."""
+
+    type = "bleu"
+
+    def __init__(self, value: Any = "", threshold: float | None = None) -> None:
+        self.value = str(value)
+        self.threshold = threshold if threshold is not None else 0.5
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        score = _compute_bleu(reference=self.value, candidate=output)
+        passed = score >= self.threshold
+        reason = f"BLEU score {score:.4f} {'>='}  threshold {self.threshold}"
+        if not passed:
+            reason = f"BLEU score {score:.4f} < threshold {self.threshold}"
+        return GradingResult(passed=passed, score=score, reason=reason)
+
+
+class RougeNAssertion:
+    """ROUGE-N score >= threshold using rouge-score library."""
+
+    type = "rouge-n"
+
+    def __init__(self, value: Any = "", threshold: float | None = None) -> None:
+        self.value = str(value)
+        self.threshold = threshold if threshold is not None else 0.75
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        if not output or not self.value:
+            score = 0.0
+        else:
+            scorer = rouge_scorer.RougeScorer(["rouge1"], use_stemmer=False)
+            scores = scorer.score(self.value, output)
+            score = scores["rouge1"].fmeasure
+
+        passed = score >= self.threshold
+        if passed:
+            reason = f"ROUGE-N score {score:.4f} >= threshold {self.threshold}"
+        else:
+            reason = f"ROUGE-N score {score:.4f} < threshold {self.threshold}"
+        return GradingResult(passed=passed, score=score, reason=reason)
+
+
+# ── Inline BLEU-4 implementation ────────────────────────────────────────────
+
+
+def _get_ngrams(tokens: list[str], n: int) -> Counter[tuple[str, ...]]:
+    """Extract n-grams from a token list."""
+    return Counter(tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1))
+
+
+def _compute_bleu(reference: str, candidate: str, max_n: int = 4) -> float:
+    """Compute BLEU score with smoothing (method 1: add 1 to numerator/denominator).
+
+    Ported from promptfoo's BLEU implementation.
+    """
+    ref_tokens = reference.lower().split()
+    cand_tokens = candidate.lower().split()
+
+    if not cand_tokens:
+        return 0.0
+    if not ref_tokens:
+        return 0.0
+
+    # Brevity penalty
+    bp = 1.0
+    if len(cand_tokens) < len(ref_tokens):
+        bp = math.exp(1.0 - len(ref_tokens) / len(cand_tokens))
+
+    # Modified precision for each n-gram order with smoothing
+    log_avg = 0.0
+    for n in range(1, max_n + 1):
+        ref_ngrams = _get_ngrams(ref_tokens, n)
+        cand_ngrams = _get_ngrams(cand_tokens, n)
+
+        # Clipped counts
+        clipped = 0
+        total = 0
+        for ngram, count in cand_ngrams.items():
+            clipped += min(count, ref_ngrams.get(ngram, 0))
+            total += count
+
+        # Smoothing: add 1 to both numerator and denominator when n > 1
+        if n == 1:
+            if total == 0:
+                return 0.0
+            precision = clipped / total
+            if precision == 0:
+                return 0.0
+        else:
+            precision = (clipped + 1) / (total + 1)
+
+        log_avg += math.log(precision) / max_n
+
+    return bp * math.exp(log_avg)

runsight_core-1.1.0/src/runsight_core/assertions/deterministic/performance.py

@@ -0,0 +1,58 @@
+"""Deterministic performance assertion plugins.
+
+Covers: cost, latency.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from runsight_core.assertions.base import AssertionContext, GradingResult
+
+
+class CostAssertion:
+    """Check that cost_usd from context is within threshold."""
+
+    type = "cost"
+
+    def __init__(self, value: Any = None, threshold: float | None = None) -> None:
+        self.value = value
+        self.threshold = threshold
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        cost = context.cost_usd
+        threshold = self.threshold if self.threshold is not None else 0.0
+        if cost <= threshold:
+            return GradingResult(
+                passed=True,
+                score=1.0,
+                reason=f"Cost ${cost:.4f} is within threshold ${threshold:.4f}",
+            )
+        return GradingResult(
+            passed=False, score=0.0, reason=f"Cost ${cost:.4f} exceeds threshold ${threshold:.4f}"
+        )
+
+
+class LatencyAssertion:
+    """Check that latency_ms from context is within threshold."""
+
+    type = "latency"
+
+    def __init__(self, value: Any = None, threshold: float | None = None) -> None:
+        self.value = value
+        self.threshold = threshold
+
+    def evaluate(self, output: str, context: AssertionContext) -> GradingResult:
+        latency = context.latency_ms
+        threshold = self.threshold if self.threshold is not None else 0.0
+        if latency <= threshold:
+            return GradingResult(
+                passed=True,
+                score=1.0,
+                reason=f"Latency {latency:.1f}ms is within threshold {threshold:.1f}ms",
+            )
+        return GradingResult(
+            passed=False,
+            score=0.0,
+            reason=f"Latency {latency:.1f}ms exceeds threshold {threshold:.1f}ms",
+        )