forgesight-eval 0.1.1__py3-none-any.whl

forgesight_eval/__init__.py

@@ -0,0 +1,18 @@
+"""ForgeSight eval — run-correlated eval scores + human feedback on the same pipeline."""
+
+from __future__ import annotations
+
+from ._config import EvalConfig, install
+from .api import record_evaluation, record_feedback
+from .model import EvaluationResult
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "EvalConfig",
+    "EvaluationResult",
+    "__version__",
+    "install",
+    "record_evaluation",
+    "record_feedback",
+]

forgesight_eval/_config.py

@@ -0,0 +1,73 @@
+"""Module config for eval emission: enabled switch, emit_as, explanation gate, score schema.
+
+``enabled`` defaults **false** — installing the package emits nothing until switched on
+(P2). ``install`` (the ``forgesight.modules`` entry point) sets it from ``modules.eval.*``;
+otherwise it lazy-loads from the SDK's layered settings + ``FORGESIGHT_EVAL_*`` env on first
+use. The optional ``score_schema`` validates score/label at the call site (fail-fast).
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+from forgesight_core.config import load_settings
+
+
+@dataclass
+class EvalConfig:
+    enabled: bool = False
+    emit_as: str = "span"  # "span" | "event"
+    capture_explanation: bool = True
+    score_schema: dict[str, dict[str, Any]] = field(default_factory=dict)
+
+
+_config: EvalConfig | None = None
+
+
+def _as_bool(value: object, default: bool) -> bool:
+    if value is None:
+        return default
+    if isinstance(value, bool):
+        return value
+    return str(value).strip().lower() in ("1", "true", "yes", "on")
+
+
+def _build(block: Mapping[str, Any]) -> EvalConfig:
+    emit_as = str(os.environ.get("FORGESIGHT_EVAL_EMIT_AS") or block.get("emit_as") or "span")
+    if emit_as not in ("span", "event"):
+        raise ValueError(f"modules.eval.emit_as must be span|event, got {emit_as!r}")
+    enabled_env = os.environ.get("FORGESIGHT_EVAL_ENABLED")
+    enabled = _as_bool(enabled_env, _as_bool(block.get("enabled"), False))
+    schema = block.get("score_schema")
+    return EvalConfig(
+        enabled=enabled,
+        emit_as=emit_as,
+        capture_explanation=_as_bool(block.get("capture_explanation"), True),
+        score_schema=dict(schema) if isinstance(schema, Mapping) else {},
+    )
+
+
+def install(config: Mapping[str, Any] | None = None) -> bool:
+    """The ``forgesight.modules`` entry point: set the eval module config. Returns ``enabled``."""
+    global _config
+    _config = _build(config or {})
+    return _config.enabled
+
+
+def get_config() -> EvalConfig:
+    """Return the active config, lazily loading from ``modules.eval`` settings if unset."""
+    global _config
+    if _config is None:
+        block = load_settings().get("modules")
+        eval_block = block.get("eval") if isinstance(block, Mapping) else None
+        _config = _build(eval_block if isinstance(eval_block, Mapping) else {})
+    return _config
+
+
+def reset_config() -> None:
+    """Clear the cached config (tests / re-configure)."""
+    global _config
+    _config = None

forgesight_eval/api.py

@@ -0,0 +1,179 @@
+"""``record_evaluation`` / ``record_feedback`` — run-correlated quality on the same pipeline.
+
+``record_evaluation`` attaches to the **current** run (ambient context) — a real-time eval
+nests as a child span under the run's still-open trace. ``record_feedback`` attaches to a
+**past** run by ``run_id`` — a standalone record carrying the id so the backend re-associates
+it. Both build an :class:`EvaluationResult`, emit it as a record with the OTel
+``gen_ai.evaluation.*`` attributes (so it streams to Langfuse scores / Phoenix evaluations /
+any OTLP sink, P4) and publish an ``EVALUATION_RECORDED`` event. Non-blocking (P6).
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Mapping
+
+from forgesight_api import EventType, Kind, LifecycleEvent, Record, RunStatus, new_trace_id
+from forgesight_core import current_context, current_run_scope, get_runtime, new_span_id
+
+from ._config import get_config
+from .model import EvaluationResult
+
+# OTel GenAI evaluation attributes (otel-semantic-conventions §4.3).
+GEN_AI_EVALUATION_NAME = "gen_ai.evaluation.name"
+GEN_AI_EVALUATION_SCORE_VALUE = "gen_ai.evaluation.score.value"
+GEN_AI_EVALUATION_SCORE_LABEL = "gen_ai.evaluation.score.label"
+GEN_AI_EVALUATION_EXPLANATION = "gen_ai.evaluation.explanation"
+# namespaced extensions (OTel defines none of these)
+FORGESIGHT_EVAL_SOURCE = "forgesight.evaluation.source"
+FORGESIGHT_EVAL_REALTIME = "forgesight.evaluation.realtime"
+FORGESIGHT_EVAL_EVALUATOR = "forgesight.evaluation.evaluator"
+FORGESIGHT_RUN_ID = "forgesight.run.id"
+
+
+def record_evaluation(
+    name: str,
+    *,
+    score: float | None = None,
+    label: str | None = None,
+    explanation: str | None = None,
+    evaluator: str | None = None,
+    run_id: str | None = None,
+    metadata: Mapping[str, object] | None = None,
+) -> None:
+    """Attach an eval (auto) to the current run, or to ``run_id`` if given. No-op if disabled."""
+    resolved_run_id = run_id or _current_run_id()
+    if resolved_run_id is None:
+        raise RuntimeError(
+            "record_evaluation has no run_id and no current run; "
+            "call it inside a run or pass run_id"
+        )
+    _validate(name, score, label)
+    _emit(
+        EvaluationResult(
+            name=name,
+            run_id=resolved_run_id,
+            score=score,
+            label=label,
+            explanation=explanation,
+            evaluator=evaluator,
+            source="auto",
+            realtime=run_id is None,
+            metadata=dict(metadata or {}),
+        )
+    )
+
+
+def record_feedback(
+    name: str,
+    *,
+    run_id: str,
+    score: float | None = None,
+    label: str | None = None,
+    comment: str | None = None,
+    source: str = "human",
+    metadata: Mapping[str, object] | None = None,
+) -> None:
+    """Attach post-hoc feedback to a past run by ``run_id``. No-op if disabled."""
+    _validate(name, score, label)
+    _emit(
+        EvaluationResult(
+            name=name,
+            run_id=run_id,
+            score=score,
+            label=label,
+            explanation=comment,
+            evaluator=None,
+            source=source,
+            realtime=False,
+            metadata=dict(metadata or {}),
+        )
+    )
+
+
+def _current_run_id() -> str | None:
+    run = current_run_scope()
+    if run is not None and run.run_id:
+        return run.run_id
+    context = current_context()
+    return context.run_id if context is not None else None
+
+
+def _validate(name: str, score: float | None, label: str | None) -> None:
+    if score is None and label is None:
+        raise ValueError(f"evaluation {name!r} must set at least one of score / label")
+    schema = get_config().score_schema.get(name)
+    if not isinstance(schema, Mapping):
+        return  # unschema'd dimension — open set, accepted unvalidated
+    kind = schema.get("type")
+    if kind == "numeric" and score is not None:
+        low, high = schema.get("min"), schema.get("max")
+        if (low is not None and score < low) or (high is not None and score > high):
+            raise ValueError(f"score {score} for {name!r} outside [{low}, {high}]")
+    if kind == "categorical" and label is not None:
+        labels = schema.get("labels") or ()
+        if label not in labels:
+            raise ValueError(f"label {label!r} for {name!r} not in {list(labels)}")
+
+
+def _emit(result: EvaluationResult) -> None:
+    config = get_config()
+    if not config.enabled:
+        return  # module installed but not switched on (P2)
+    runtime = get_runtime()
+    context = current_context()
+    if result.realtime and context is not None:
+        trace_id, parent = context.trace_id, context.current_span_id
+    else:
+        trace_id, parent = new_trace_id(), None  # post-hoc: standalone, re-associated by run_id
+
+    now = time.time_ns()
+    record = Record(
+        kind=Kind.STEP,
+        run_id=result.run_id,
+        trace_id=trace_id,
+        span_id=new_span_id(),
+        parent_span_id=parent,
+        name=f"evaluation {result.name}",
+        status=RunStatus.OK,
+        start_unix_nanos=now,
+        end_unix_nanos=now,
+        attributes=_attributes(result, config.capture_explanation),
+    )
+    runtime.emit_record(record)
+    runtime.emit_event(
+        LifecycleEvent(
+            type=EventType.EVALUATION_RECORDED,
+            run_id=result.run_id,
+            unix_nanos=now,
+            record=record,
+            trace_id=trace_id,
+            span_id=record.span_id,
+        )
+    )
+
+
+def _attributes(result: EvaluationResult, capture_explanation: bool) -> dict[str, object]:
+    attrs: dict[str, object] = dict(result.metadata)
+    attrs[GEN_AI_EVALUATION_NAME] = result.name
+    attrs[FORGESIGHT_RUN_ID] = result.run_id
+    attrs[FORGESIGHT_EVAL_SOURCE] = result.source
+    attrs[FORGESIGHT_EVAL_REALTIME] = result.realtime
+    if result.score is not None:
+        attrs[GEN_AI_EVALUATION_SCORE_VALUE] = result.score
+    if result.label is not None:
+        attrs[GEN_AI_EVALUATION_SCORE_LABEL] = result.label
+    if result.evaluator is not None:
+        attrs[FORGESIGHT_EVAL_EVALUATOR] = result.evaluator
+    # explanation/comment is free text → gated by capture_explanation AND the global
+    # content-capture switch (P7); dropped if either is off.
+    if result.explanation is not None and capture_explanation and _content_capture_on():
+        attrs[GEN_AI_EVALUATION_EXPLANATION] = result.explanation
+    return attrs
+
+
+def _content_capture_on() -> bool:
+    try:
+        return bool(get_runtime().config.capture_content)
+    except Exception:  # pragma: no cover - runtime always present in practice
+        return False

forgesight_eval/model.py

@@ -0,0 +1,28 @@
+"""``EvaluationResult`` — the run-correlated quality signal (feat-021).
+
+The same value type for an automated eval (LLM-as-judge / Ragas / DeepEval) and a post-hoc
+human thumbs-up/down. ``realtime`` distinguishes "attached during the run" from "arrived
+later by ``run_id``"; ``source`` distinguishes ``auto`` from ``human``. At least one of
+``score`` / ``label`` must be set (enforced at the call site). Experimental within 0.x.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from types import MappingProxyType
+
+_EMPTY: Mapping[str, object] = MappingProxyType({})
+
+
+@dataclass(frozen=True, slots=True)
+class EvaluationResult:
+    name: str
+    run_id: str
+    score: float | None = None
+    label: str | None = None
+    explanation: str | None = None
+    evaluator: str | None = None
+    source: str = "auto"  # "auto" (eval) | "human" (feedback)
+    realtime: bool = True  # True if attached during the run, else post-hoc
+    metadata: Mapping[str, object] = field(default_factory=lambda: _EMPTY)

forgesight_eval-0.1.1.dist-info/METADATA

@@ -0,0 +1,88 @@
+Metadata-Version: 2.4
+Name: forgesight-eval
+Version: 0.1.1
+Summary: ForgeSight eval — run-correlated eval scores + human feedback on the telemetry pipeline.
+Project-URL: Homepage, https://github.com/Scaffoldic/forgesight
+Project-URL: Repository, https://github.com/Scaffoldic/forgesight
+Project-URL: Issues, https://github.com/Scaffoldic/forgesight/issues
+Project-URL: Changelog, https://github.com/Scaffoldic/forgesight/blob/main/docs/releases/v0.1.md
+Author: kjoshi
+License-Expression: Apache-2.0
+Keywords: ai-agents,evaluation,feedback,forgesight,llm-as-judge,observability
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: System :: Monitoring
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Requires-Dist: forgesight-core
+Description-Content-Type: text/markdown
+
+# forgesight-eval
+
+Run-correlated **eval scores and human feedback** for [ForgeSight](https://github.com/Scaffoldic/forgesight).
+Quality joins cost and structure on the *same* `run_id`, on the *same* pipeline, in the *same*
+backends — Langfuse scores, Phoenix evaluations, any OTLP sink — for two function calls.
+
+```bash
+pip install forgesight-eval
+```
+
+```python
+from forgesight import telemetry
+from forgesight_eval import record_evaluation, record_feedback
+
+with telemetry.agent_run("rag-answerer", metadata={"prompt_version": "v7"}) as run:
+    answer = await answer_question(...)
+    # an automated eval (LLM-as-judge / Ragas / DeepEval) → one call, correlated to the live run
+    record_evaluation("faithfulness", score=0.91, label="pass",
+                      explanation="All claims grounded.", evaluator="ragas")
+
+# hours later, from a webhook — post-hoc human feedback, by run_id:
+def on_thumbs_down(run_id: str, comment: str) -> None:
+    record_feedback("user_satisfaction", run_id=run_id, label="thumbs_down",
+                    score=0.0, comment=comment, source="human")
+```
+
+## How it works
+
+- **`record_evaluation`** attaches to the **current** run (ambient context) — a real-time eval
+  nests as a child span under the run's open trace. **`record_feedback`** attaches to a **past**
+  run by `run_id` — a standalone record carrying the id so the backend re-associates it.
+- Both emit the OTel **`gen_ai.evaluation.*`** attributes (`name` / `score.value` / `score.label`
+  / `explanation`) plus `forgesight.evaluation.*` extensions (`source`, `realtime`, `evaluator`),
+  and publish an `EVALUATION_RECORDED` lifecycle event. Backends that speak the convention
+  (Langfuse, Phoenix) display them as scores/evaluations with **no per-backend code** (P4).
+- **Non-blocking** (P6): both build a record and enqueue — no network I/O on the agent or the
+  webhook. **Secure by default** (P7): `explanation` / `comment` text is gated by
+  `capture_explanation` *and* the global content-capture switch.
+
+## Configuration
+
+```yaml
+modules:
+  eval:
+    enabled: true              # master switch (default false — install ≠ active)
+    emit_as: "span"            # span | event
+    capture_explanation: true  # still gated by the global content-capture switch (P7)
+    score_schema:              # optional — validates score/label at the call site
+      faithfulness:      { type: "numeric", min: 0.0, max: 1.0 }
+      user_satisfaction: { type: "categorical", labels: ["thumbs_up", "thumbs_down"] }
+```
+
+At least one of `score` / `label` must be set. Schema'd dimensions are validated (numeric range,
+categorical membership); un-schema'd dimensions are accepted unvalidated (open set).
+
+## Out of scope
+
+Running the evaluations (judges/metrics live in the agent or a library — this is the transport),
+a dashboard, and score storage/aggregation (the backend's job).
+
+## License
+
+Apache-2.0

forgesight_eval-0.1.1.dist-info/RECORD

@@ -0,0 +1,9 @@
+forgesight_eval/__init__.py,sha256=FAFbys5zZA3Eg3XyuIR1Wcwm4mePSg3CDvnFuVPAX4M,420
+forgesight_eval/_config.py,sha256=33x9TQDzKLQP8HVj-EEh9OrEvlNbbxMnWunZk9Gtros,2572
+forgesight_eval/api.py,sha256=sOqgjQoe4o3c3blxiuoTgqR1rvfEsdjmYEm8z-DoTO8,6603
+forgesight_eval/model.py,sha256=CsL30_ZnRfAgKpxUuQIhxcIXpEQ5nHTDBZYeD7BEwF0,1071
+forgesight_eval/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+forgesight_eval-0.1.1.dist-info/METADATA,sha256=ZOZftAc-qhO2DW2oX5fGhfjy5o34a_ot9zROx5rNN34,4098
+forgesight_eval-0.1.1.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+forgesight_eval-0.1.1.dist-info/entry_points.txt,sha256=JqpwMBH6oLdtloqPPX1nJJgJBDFFHTfpFMAJaBFjmgw,52
+forgesight_eval-0.1.1.dist-info/RECORD,,

forgesight_eval-0.1.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

forgesight_eval-0.1.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[forgesight.modules]
+eval = forgesight_eval:install