nemo-platform-plugin 0.1.0__py3-none-any.whl

nemo_platform/beta/evaluator/structured_output.py

@@ -0,0 +1,184 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+from enum import Enum
+
+from jsonschema.exceptions import SchemaError
+from jsonschema.validators import validator_for
+from pydantic import BaseModel, Field, field_validator
+
+from nemo_platform.beta.evaluator.enums import ModelFormat
+from nemo_platform.beta.evaluator.inference import InferenceFn, PreprocessRequest, deep_merge
+from nemo_platform.beta.evaluator.values import Model
+
+
+class StructuredOutputMode(str, Enum):
+    OPENAI_RESPONSE_FORMAT = "openai_response_format"
+    ROOT_GUIDED_JSON = "root_guided_json"
+    NVEXT_GUIDED_JSON = "nvext_guided_json"
+    UNSUPPORTED = "unsupported"
+
+
+class StructuredOutput(BaseModel):
+    name: str | None = None
+    json_schema: dict = Field(alias="schema")
+    strict: bool = False
+
+    @field_validator("json_schema")
+    @classmethod
+    def validate_json_schema(cls, value: dict):
+        validator = validator_for(value)
+        validator.check_schema(value)
+        return value
+
+
+class InferenceStructuredOutput(PreprocessRequest):
+    """Format structured output request parameters based on provider mode."""
+
+    def __init__(self, mode: StructuredOutputMode, structured_output: dict):
+        if not structured_output:
+            raise ValueError("structured_output cannot be empty")
+        try:
+            output = StructuredOutput(**structured_output)
+            self._json_schema = output.json_schema
+            self._strict = output.strict
+            self.mode = mode
+            self.inference_param = self._build_inference_param(mode)
+        except SchemaError as e:
+            raise ValueError("structured output contains invalid JSON schema") from e
+
+    @property
+    def json_schema(self) -> dict:
+        return self._json_schema
+
+    def set_mode(self, mode: StructuredOutputMode) -> None:
+        self.mode = mode
+        self.inference_param = self._build_inference_param(mode)
+
+    def _build_inference_param(self, mode: StructuredOutputMode) -> dict:
+        if mode == StructuredOutputMode.OPENAI_RESPONSE_FORMAT:
+            return {
+                "response_format": {
+                    "type": "json_schema",
+                    "json_schema": {
+                        "name": "structured_output",
+                        "schema": self._json_schema,
+                        "strict": self._strict,
+                    },
+                }
+            }
+        if mode == StructuredOutputMode.ROOT_GUIDED_JSON:
+            return {"extra_body": {"guided_json": self._json_schema}}
+        if mode == StructuredOutputMode.NVEXT_GUIDED_JSON:
+            return {"extra_body": {"nvext": {"guided_json": self._json_schema}}}
+        if mode == StructuredOutputMode.UNSUPPORTED:
+            return {}
+        raise ValueError(f"Unsupported structured output mode: {mode}")
+
+    def _apply_fallback_instruction(self, request: dict) -> dict:
+        schema_str = json.dumps(self._json_schema, separators=(",", ":"))
+        instruction = f"Return ONLY valid JSON and ensure it matches this JSON schema exactly: {schema_str}"
+        if request.get("messages"):
+            msg = request["messages"][0]
+            if msg.get("role") == "system":
+                request["messages"][0]["content"] = f"{instruction} {msg['content']}"
+            else:
+                request["messages"].insert(0, {"role": "system", "content": instruction})
+        elif request.get("prompt"):
+            request["prompt"] = f"{instruction} {request['prompt']}"
+        return request
+
+    def preprocess(self, request: dict, id: str | None = None) -> dict:
+        _ = id  # Required by preprocess hook interface.
+        if self.mode == StructuredOutputMode.UNSUPPORTED:
+            return self._apply_fallback_instruction(request)
+        # Use merge instead of update to avoid overwriting nested dicts
+        return deep_merge(request, self.inference_param)
+
+
+def default_structured_output_mode(format: str) -> StructuredOutputMode:
+    if format == ModelFormat.OPEN_AI:
+        return StructuredOutputMode.OPENAI_RESPONSE_FORMAT
+    if format == ModelFormat.NVIDIA_NIM:
+        # Backward-compatible default before preflight detection overrides this.
+        return StructuredOutputMode.NVEXT_GUIDED_JSON
+    raise ValueError(f"Unsupported structured output format: {format}")
+
+
+def _looks_like_unsupported_guided_json_error(message: str) -> bool:
+    lowered = message.lower()
+    signatures = (
+        "guided_json is unsupported",
+        "unexpected keyword argument 'guided_json'",
+        "unexpected keyword argument 'nvext'",
+        "extra_forbidden",
+        "extra inputs are not permitted",
+    )
+    if any(sig in lowered for sig in signatures):
+        return "guided_json" in lowered or "nvext" in lowered or "extra_body" in lowered
+    return False
+
+
+def _extract_chat_content(response: dict) -> str | None:
+    choices = response.get("choices")
+    if not isinstance(choices, list) or not choices:
+        return None
+    msg = choices[0].get("message", {})
+    content = msg.get("content")
+    return content if isinstance(content, str) else None
+
+
+def _is_probe_valid_json(content: str, probe_schema: dict) -> bool:
+    try:
+        obj = json.loads(content)
+    except (TypeError, ValueError):
+        return False
+    if not isinstance(obj, dict):
+        return False
+    try:
+        validator = validator_for(probe_schema)
+        validator.check_schema(probe_schema)
+        validator(probe_schema).validate(obj)
+        return True
+    except Exception:
+        return False
+
+
+async def detect_structured_output_mode(
+    *,
+    format: str,
+    model: Model,
+    inference_fn: InferenceFn,
+    api_key: str | None,
+    probe_schema: dict,
+) -> StructuredOutputMode:
+    """Detect working structured output mode for the given model/format."""
+    if format == ModelFormat.OPEN_AI:
+        return StructuredOutputMode.OPENAI_RESPONSE_FORMAT
+    if format != ModelFormat.NVIDIA_NIM:
+        return StructuredOutputMode.UNSUPPORTED
+
+    probe_message = "Return ONLY a JSON object that matches the provided schema exactly. No prose or code fences."
+    base_request = {
+        "messages": [{"role": "user", "content": probe_message}],
+        "temperature": 0,
+        "max_tokens": 128,
+    }
+    candidates: list[tuple[StructuredOutputMode, dict]] = [
+        (StructuredOutputMode.ROOT_GUIDED_JSON, {"extra_body": {"guided_json": probe_schema}}),
+        (StructuredOutputMode.NVEXT_GUIDED_JSON, {"extra_body": {"nvext": {"guided_json": probe_schema}}}),
+    ]
+    for mode, structured_param in candidates:
+        try:
+            response = await inference_fn(model, {**base_request, **structured_param}, 1, api_key=api_key)
+            content = _extract_chat_content(response)
+            if content and _is_probe_valid_json(content, probe_schema):
+                return mode
+        except Exception as e:
+            if _looks_like_unsupported_guided_json_error(str(e)):
+                continue
+            # Probe failures should not abort evaluation startup. If no mode works,
+            # caller will fall back to prompt-level strict JSON instruction.
+            continue
+    return StructuredOutputMode.UNSUPPORTED

nemo_platform/beta/evaluator/templates.py

@@ -0,0 +1,115 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Template rendering helpers for evaluator SDK runtime."""
+
+# Migrated from: services/evaluator/src/nmp/evaluator/app/templates.py
+
+import json
+from typing import Any
+
+from jinja2 import StrictUndefined
+from jinja2.sandbox import SandboxedEnvironment
+
+env = SandboxedEnvironment(undefined=StrictUndefined)
+
+# Preserve native Python values for bare expressions like `{{ item.score }}`.
+# Full template rendering always returns strings, which breaks structured payloads.
+_EXPR_PREFIX = "{{"
+_EXPR_SUFFIX = "}}"
+
+
+def _is_single_expression_template(template: str) -> bool:
+    """Check whether a template is exactly one Jinja expression.
+
+    Args:
+        template: Raw template string.
+
+    Returns:
+        ``True`` when the string is a single ``{{ ... }}`` expression with no
+        statement blocks; otherwise ``False``.
+    """
+    stripped = template.strip()
+    return (
+        stripped.startswith(_EXPR_PREFIX)
+        and stripped.endswith(_EXPR_SUFFIX)
+        and stripped.count(_EXPR_PREFIX) == 1
+        and stripped.count(_EXPR_SUFFIX) == 1
+        and "{%" not in stripped
+        and "%}" not in stripped
+    )
+
+
+def _identifier_kwargs(context: dict) -> dict:
+    """Filter context keys that can be passed to ``compile_expression`` kwargs.
+
+    Args:
+        context: Template rendering context.
+
+    Returns:
+        Dictionary containing only string keys that are valid Python identifiers.
+    """
+    # compile_expression() only accepts keyword arguments, so keys like
+    # `foo-bar` need to stay accessible through `item`/`sample` instead.
+    return {k: v for k, v in context.items() if isinstance(k, str) and k.isidentifier()}
+
+
+def render_template(template: str | dict | list, context: dict) -> Any:
+    """Render strings, dicts, or lists using sandboxed Jinja evaluation.
+
+    For bare expression templates (for example ``{{ item.payload }}``), the
+    function uses ``compile_expression`` to preserve native value types instead
+    of forcing string output.
+
+    Args:
+        template: Template payload to render.
+        context: Variables available to the Jinja runtime.
+
+    Returns:
+        Rendered value preserving dict/list structure and native expression types.
+
+    Raises:
+        jinja2.UndefinedError: If the template references a missing variable.
+        json.JSONDecodeError: If ``tojson`` output is not valid JSON.
+    """
+    if isinstance(template, dict):
+        return {k: render_template(v, context) for k, v in template.items()}
+    if isinstance(template, list):
+        return [render_template(v, context) for v in template]
+    if isinstance(template, str):
+        if _is_single_expression_template(template):
+            # Use the expression compiler here so `{{ some_dict }}` returns a dict
+            # instead of a stringified representation.
+            expr = template.strip()[len(_EXPR_PREFIX) : -len(_EXPR_SUFFIX)].strip()
+            compiled = env.compile_expression(expr, undefined_to_none=False)
+            result = compiled(**_identifier_kwargs(context))
+            if isinstance(result, StrictUndefined):
+                str(result)
+            if "tojson" in template and isinstance(result, str):
+                return json.loads(result)
+            return result
+
+        rendered_str = env.from_string(template).render(context)
+        if "tojson" in template:
+            return json.loads(rendered_str)
+        return rendered_str
+    return template
+
+
+def render_request(template: str | dict, context: dict) -> dict:
+    """Render a request payload and normalize string output into prompt dicts.
+
+    Args:
+        template: String or dictionary request template.
+        context: Variables available to the Jinja runtime.
+
+    Returns:
+        Dictionary request payload. String templates are wrapped as
+        ``{"prompt": rendered_text}``.
+    """
+    request = render_template(template, context=context)
+    if isinstance(request, str):
+        request = {"prompt": request}
+    if isinstance(request, list):
+        raise TypeError("Request template must not produce a list output.")
+    return request

nemo_platform/beta/evaluator/values/__init__.py

@@ -0,0 +1,162 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Public value types for evaluator SDK runtime."""
+
+from nemo_platform.beta.evaluator.metrics.protocol import (
+    BooleanValue,
+    CandidateOutput,
+    ContinuousScore,
+    DatasetRow,
+    DiscreteScore,
+    Label,
+    MetricDescriptor,
+    MetricInput,
+    MetricOutput,
+    MetricOutputSpec,
+    MetricResult,
+)
+from nemo_platform.beta.evaluator.values.agents import Agent
+from nemo_platform.beta.evaluator.values.common import SecretRef, SupportedJobTypes
+from nemo_platform.beta.evaluator.values.dataset_schemas import (
+    FieldMapping,
+    InputSchema,
+)
+from nemo_platform.beta.evaluator.values.datasets import DatasetInput, DatasetRows
+from nemo_platform.beta.evaluator.values.metrics import (
+    BLEU,
+    F1,
+    ROUGE,
+    AgentGoalAccuracy,
+    AnswerAccuracy,
+    ContextEntityRecall,
+    ContextPrecision,
+    ContextRecall,
+    ContextRelevance,
+    ExactMatch,
+    Faithfulness,
+    LLMJudge,
+    MetricBase,
+    NemoAgentToolkitRemote,
+    NoiseSensitivity,
+    NumberCheck,
+    Remote,
+    ResponseGroundedness,
+    ResponseRelevancy,
+    StringCheck,
+    ToolCallAccuracy,
+    ToolCalling,
+    TopicAdherence,
+)
+from nemo_platform.beta.evaluator.values.models import Model, ReasoningParams
+from nemo_platform.beta.evaluator.values.params import (
+    InferenceParams,
+    RunConfig,
+    RunConfigOnline,
+    RunConfigOnlineModel,
+)
+from nemo_platform.beta.evaluator.values.results import (
+    AggregatedMetricResult,
+    AggregateFieldName,
+    AggregateRangeScore,
+    AggregateRubricScore,
+    AggregateScore,
+    AggregateScoreBase,
+    DefaultAggregateFieldName,
+    EvaluationResult,
+    Histogram,
+    HistogramBin,
+    MetricScore,
+    Percentiles,
+    RowScore,
+    RubricScoreStat,
+    RubricScoreValue,
+    SampleResult,
+    ScoreStats,
+)
+from nemo_platform.beta.evaluator.values.scores import (
+    JSONScoreParser,
+    RangeScore,
+    RegexScoreParser,
+    RemoteScore,
+    Rubric,
+    RubricScore,
+    Score,
+    score_discriminator,
+)
+
+__all__ = [
+    "Agent",
+    "AggregateFieldName",
+    "AggregatedMetricResult",
+    "AggregateRangeScore",
+    "AggregateRubricScore",
+    "AggregateScore",
+    "AggregateScoreBase",
+    "BooleanValue",
+    "CandidateOutput",
+    "ContinuousScore",
+    "DatasetRow",
+    "DatasetRows",
+    "DefaultAggregateFieldName",
+    "DiscreteScore",
+    "RunConfig",
+    "RunConfigOnline",
+    "RunConfigOnlineModel",
+    "FieldMapping",
+    "Histogram",
+    "HistogramBin",
+    "InferenceParams",
+    "JSONScoreParser",
+    "Label",
+    "MetricDescriptor",
+    "MetricInput",
+    "MetricOutput",
+    "MetricOutputSpec",
+    "MetricResult",
+    "MetricScore",
+    "Model",
+    "DatasetInput",
+    "EvaluationResult",
+    "Percentiles",
+    "RangeScore",
+    "ReasoningParams",
+    "InputSchema",
+    "RegexScoreParser",
+    "RemoteScore",
+    "RowScore",
+    "Rubric",
+    "RubricScore",
+    "RubricScoreStat",
+    "RubricScoreValue",
+    "SampleResult",
+    "Score",
+    "ScoreStats",
+    "SecretRef",
+    "SupportedJobTypes",
+    "score_discriminator",
+    # Metrics
+    "AgentGoalAccuracy",
+    "AnswerAccuracy",
+    "BLEU",
+    "ContextEntityRecall",
+    "ContextPrecision",
+    "ContextRecall",
+    "ContextRelevance",
+    "ExactMatch",
+    "F1",
+    "Faithfulness",
+    "LLMJudge",
+    "MetricBase",
+    "NemoAgentToolkitRemote",
+    "NoiseSensitivity",
+    "NumberCheck",
+    "Remote",
+    "ResponseGroundedness",
+    "ResponseRelevancy",
+    "ROUGE",
+    "StringCheck",
+    "ToolCallAccuracy",
+    "ToolCalling",
+    "TopicAdherence",
+]

nemo_platform/beta/evaluator/values/agents.py

@@ -0,0 +1,84 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Agent-related value types."""
+
+from __future__ import annotations
+
+import os
+from functools import cached_property
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+from nemo_platform.beta.evaluator.enums import AgentFormat
+from nemo_platform.beta.evaluator.values.common import SecretRef
+
+
+class Agent(BaseModel):
+    """Agent definition for inference in online evaluation jobs.
+
+    An agent is an endpoint that accepts a request and returns a response,
+    potentially with a trajectory. Two formats are supported:
+
+    - ``generic``: configurable HTTP POST with Jinja-templated body and
+      JSONPath extraction for response and trajectory.
+    - ``nemo_agent_toolkit``: NeMo Agent Toolkit SSE streaming protocol
+      (``/generate/full?filter_steps=none``).
+    """
+
+    # TODO: Much of this is duplicated between agent and model. Once we have aligned on model defination.
+    # the duplication can be removed by defining EndPoint class and reusing it across both model and agent.
+    model_config = ConfigDict(extra="forbid")
+
+    url: str = Field(description="Base URL of the agent endpoint.")
+    name: str = Field(description="Agent name / identifier.")
+    format: Literal[AgentFormat.GENERIC, AgentFormat.NEMO_AGENT_TOOLKIT] = Field(
+        default=AgentFormat.GENERIC, description="Agent format that determines the execution path."
+    )
+    api_key_secret: SecretRef | None = Field(
+        default=None,
+        description="API key secret reference for the agent. Format: workspace/secret_name or secret_name within the job workspace.",
+    )
+
+    # Generic agent fields
+    body: dict | None = Field(
+        default=None,
+        description="Jinja template for the request payload. Required for generic agents.",
+    )
+    response_path: str | None = Field(
+        default=None,
+        description="JSONPath expression to extract the response text from the agent's response body. Required for generic agents.",
+    )
+    trajectory_path: str | None = Field(
+        default=None,
+        description="JSONPath expression to extract the trajectory from the agent's response body. Optional.",
+    )
+
+    # TODO: When agent is type NAT, prefill body, response_path and trajectory_path depending on url ends with generate or generate/full.
+    @model_validator(mode="after")
+    def _validate_generic_fields(self) -> Agent:
+        if self.format == AgentFormat.GENERIC:
+            if self.body is None:
+                raise ValueError("'body' is required when agent format is 'generic'.")
+            if self.response_path is None:
+                raise ValueError("'response_path' is required when agent format is 'generic'.")
+        return self
+
+    @cached_property
+    def api_key_env(self) -> str | None:
+        if self.api_key_secret:
+            env_name = self.api_key_secret.root
+            if env_name[0].isdigit():
+                env_name = f"_{env_name}"
+            return env_name.replace("-", "_").replace("/", "_")
+        return None
+
+    @cached_property
+    def api_key(self) -> str | None:
+        if self.api_key_secret:
+            api_key_env = self.api_key_env
+            if api_key_env is None:
+                raise ValueError("api_key_env must be set when api_key_secret is provided")
+            return os.getenv(api_key_env)
+        return None

nemo_platform/beta/evaluator/values/common.py

@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Common value types used throughout evaluator SDK runtime."""
+
+# Migrated from: services/evaluator/src/nmp/evaluator/app/values/common.py
+
+from enum import Enum
+
+from pydantic import Field, RootModel
+
+
+class SupportedJobTypes(str, Enum):
+    ONLINE = "online"
+    OFFLINE = "offline"
+    RETRIEVER = "retriever"
+
+
+class SecretRef(RootModel):
+    root: str = Field(
+        description="Reference to a platform secret or local environment variable. Format: 'secret_name' (uses request workspace) or 'workspace/secret_name' (explicit workspace).",
+        pattern=r"^[A-Za-z0-9_-]+(/[A-Za-z0-9_-]+)?$",
+        examples=[
+            "my-secret",
+            "my-workspace/my-secret",
+            "NVIDIA_API_KEY",
+        ],
+    )

nemo_platform/beta/evaluator/values/dataset_schemas.py

@@ -0,0 +1,112 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Public Pydantic models for canonical evaluator schema and dataset column mapping."""
+
+from __future__ import annotations
+
+from typing import Annotated, Self
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+_KNOWN_BINDING_FIELDS = (
+    "input",
+    "output",
+    "context",
+    "reference",
+    "trajectory",
+    "messages",
+    "tool_calls",
+    "tools",
+)
+
+_FIELD_MAPPING_PATH_PATTERN = r"^[^\[\]]*$"
+_FieldMappingPath = Annotated[str, Field(pattern=_FIELD_MAPPING_PATH_PATTERN, min_length=1)]
+
+
+class InputSchema(BaseModel):
+    model_config = ConfigDict(serialize_by_alias=True)
+
+    schema_: dict = Field(
+        alias="schema",
+        description=(
+            "Canonical evaluator input schema expressed as JSON Schema. "
+            "This describes the normalized template context required by the metric, "
+            "not the raw dataset row shape."
+        ),
+    )
+
+    @model_validator(mode="after")
+    def validate_schema(self) -> Self:
+        from nemo_platform.beta.evaluator.dataset_schemas.common import validate_json_schema
+
+        validate_json_schema(self.schema_)
+        return self
+
+
+class _FieldMappingBase(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    input: _FieldMappingPath | None = Field(
+        default=None, description="Binding for the canonical 'input' evaluator field."
+    )
+    output: _FieldMappingPath | None = Field(
+        default=None, description="Binding for the canonical 'output' evaluator field."
+    )
+    context: _FieldMappingPath | None = Field(
+        default=None, description="Binding for the canonical 'context' evaluator field."
+    )
+    reference: _FieldMappingPath | None = Field(
+        default=None,
+        description="Binding for the canonical 'reference' evaluator field.",
+    )
+    trajectory: _FieldMappingPath | None = Field(
+        default=None,
+        description="Binding for the canonical 'trajectory' evaluator field.",
+    )
+    messages: _FieldMappingPath | None = Field(
+        default=None,
+        description="Binding for the canonical 'messages' evaluator field.",
+    )
+    tool_calls: _FieldMappingPath | None = Field(
+        default=None,
+        description="Binding for the canonical 'tool_calls' evaluator field.",
+    )
+    tools: _FieldMappingPath | None = Field(
+        default=None, description="Binding for the canonical 'tools' evaluator field."
+    )
+    custom: dict[str, _FieldMappingPath] = Field(
+        default_factory=dict,
+        description="Additional evaluator field bindings keyed by canonical field name.",
+    )
+
+    @model_validator(mode="after")
+    def validate_custom_keys(self) -> Self:
+        duplicates = sorted(set(self.custom).intersection(_KNOWN_BINDING_FIELDS))
+        if duplicates:
+            raise ValueError(f"custom binding keys overlap with reserved evaluator fields: {duplicates}")
+        return self
+
+    def mapping(self) -> dict[str, str]:
+        result = {name: value for name in _KNOWN_BINDING_FIELDS if (value := getattr(self, name)) is not None}
+        result.update(self.custom)
+        return result
+
+
+class FieldMapping(_FieldMappingBase):
+    """Maps canonical evaluator fields to raw dataset column paths.
+    Example: {'input': 'question', 'output': 'answer', 'reference': 'gold', 'trajectory': 'steps'}
+    """
+
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="after")
+    def validate_supported_dataset_paths(self) -> Self:
+        unsupported = sorted(
+            canonical_name
+            for canonical_name, dataset_path in self.mapping().items()
+            if "[" in dataset_path or "]" in dataset_path
+        )
+        if unsupported:
+            raise ValueError(f"array path segments are not supported for column mappings: {unsupported}")
+        return self