snodo-validator-tool-jev 0.23.3__tar.gz

snodo_validator_tool_jev-0.23.3/PKG-INFO

@@ -0,0 +1,28 @@
+Metadata-Version: 2.4
+Name: snodo-validator-tool-jev
+Version: 0.23.3
+Summary: TypeSafe Jev decision tool for Snodo validators
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: snodo-engine==0.23.3
+Requires-Dist: httpx>=0.27.0
+
+# Snodo TypeSafe Jev validator tool
+
+An optional plugin that adds the `decide` function to opted-in LLM validators.
+Jev returns typed answers and calibrated probabilities; the LLM remains
+responsible for writing the validator verdict.
+
+Install it into the same Python environment as Snodo and configure
+`validator_tools: [{name: decide}]` on an LLM validator. The tool accepts
+`context` as `diff`, `spec`, or `files` (with up to a small number of repository
+relative file paths), and a list of named `questions` using `noul`, `choice`, or
+`score` types. Choice criteria map option names to descriptions; score criteria
+are ordered level labels.
+
+Per-tool protocol options are `model` (default `jev-latest`), `api_key_env`
+(default `TYPESAFE_API_KEY`), `base_url` (default `https://api.typesafe.ai`),
+`timeout_seconds`, and `max_input_chars`. The key is read only from the named
+environment variable. Each result line reports the typed answer and
+probabilities; usage and any context truncation follow. Jev pricing is $0.042
+per million input tokens; its maximum context length is unpublished.

snodo_validator_tool_jev-0.23.3/README.md

@@ -0,0 +1,19 @@
+# Snodo TypeSafe Jev validator tool
+
+An optional plugin that adds the `decide` function to opted-in LLM validators.
+Jev returns typed answers and calibrated probabilities; the LLM remains
+responsible for writing the validator verdict.
+
+Install it into the same Python environment as Snodo and configure
+`validator_tools: [{name: decide}]` on an LLM validator. The tool accepts
+`context` as `diff`, `spec`, or `files` (with up to a small number of repository
+relative file paths), and a list of named `questions` using `noul`, `choice`, or
+`score` types. Choice criteria map option names to descriptions; score criteria
+are ordered level labels.
+
+Per-tool protocol options are `model` (default `jev-latest`), `api_key_env`
+(default `TYPESAFE_API_KEY`), `base_url` (default `https://api.typesafe.ai`),
+`timeout_seconds`, and `max_input_chars`. The key is read only from the named
+environment variable. Each result line reports the typed answer and
+probabilities; usage and any context truncation follow. Jev pricing is $0.042
+per million input tokens; its maximum context length is unpublished.

snodo_validator_tool_jev-0.23.3/pyproject.toml

@@ -0,0 +1,18 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "snodo-validator-tool-jev"
+version = "0.23.3"
+description = "TypeSafe Jev decision tool for Snodo validators"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = ["snodo-engine==0.23.3", "httpx>=0.27.0"]
+
+[project.entry-points."snodo.validator_tools"]
+decide = "snodo_validator_tool_jev:tool"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+namespaces = false

snodo_validator_tool_jev-0.23.3/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

snodo_validator_tool_jev-0.23.3/src/snodo_validator_tool_jev/__init__.py

@@ -0,0 +1,173 @@
+"""TypeSafe Jev decision tool for Snodo LLM validators."""
+
+from __future__ import annotations
+
+import json
+import os
+from typing import Any, Mapping
+
+import httpx
+
+from snodo.validators.tool_registry import ValidatorTool, ValidatorToolContext
+
+_MAX_FILES = 5
+
+def handle_decide(arguments: Mapping[str, Any], context: ValidatorToolContext) -> str:
+    """Send validated questions and selected context to Jev; return safe text."""
+    key_name = str(context.tool_options.get("api_key_env", "TYPESAFE_API_KEY"))
+    api_key = os.environ.get(key_name)
+    if not api_key:
+        return f"TypeSafe Jev unavailable: environment variable {key_name} is not set."
+    try:
+        questions = _normalize_questions(arguments.get("questions"))
+        state = _get_context(arguments, context)
+        max_chars = max(1, int(context.tool_options.get("max_input_chars", 50000)))
+        truncated = len(state) > max_chars
+        state = state[:max_chars]
+        timeout = float(context.tool_options.get("timeout_seconds", 30))
+        base_url = str(context.tool_options.get("base_url", "https://api.typesafe.ai")).rstrip("/")
+        model = str(context.tool_options.get("model", "jev-latest"))
+        response = httpx.post(
+            f"{base_url}/v1/systemone",
+            headers={"Authorization": f"Bearer {api_key}"},
+            json={"model": model, "state": state, "questions": questions},
+            timeout=timeout,
+        )
+        response.raise_for_status()
+        payload = response.json()
+        if not isinstance(payload, dict) or not isinstance(payload.get("answers"), dict):
+            return "TypeSafe Jev error: malformed response; expected an answers object."
+        return _render(payload, questions, truncated)
+    except httpx.TimeoutException:
+        return "TypeSafe Jev error: request timed out. Check the endpoint or increase timeout_seconds."
+    except httpx.HTTPStatusError as exc:
+        return f"TypeSafe Jev error: HTTP {exc.response.status_code} from the decision API."
+    except httpx.HTTPError:
+        return "TypeSafe Jev error: could not complete the HTTP request. Check the endpoint and network."
+    except (ValueError, TypeError, OSError, json.JSONDecodeError) as exc:
+        # Do not include exception text: paths or HTTP internals must not leak secrets.
+        return f"TypeSafe Jev error: {type(exc).__name__}; check the tool arguments and options."
+
+
+def _normalize_questions(raw: Any) -> dict[str, dict[str, Any]]:
+    if not isinstance(raw, list) or not raw:
+        raise ValueError("questions must be a non-empty list")
+    normalized: dict[str, dict[str, Any]] = {}
+    for question in raw:
+        if not isinstance(question, Mapping):
+            raise ValueError("each question must be an object")
+        name, kind, instructions = question.get("name"), question.get("type"), question.get("instructions")
+        if not all(isinstance(value, str) and value.strip() for value in (name, kind, instructions)):
+            raise ValueError("question name, type, and instructions are required")
+        if name in normalized or kind not in {"noul", "choice", "score"}:
+            raise ValueError("question names must be unique and types must be noul, choice, or score")
+        item: dict[str, Any] = {"type": kind, "instructions": instructions}
+        criteria = question.get("criteria")
+        if kind == "choice":
+            if not isinstance(criteria, Mapping) or not criteria:
+                raise ValueError("choice questions require criteria as an option-description map")
+            item["criteria"] = dict(criteria)
+        elif kind == "score":
+            if not isinstance(criteria, list) or not criteria:
+                raise ValueError("score questions require criteria as an ordered list of levels")
+            item["criteria"] = criteria
+        elif criteria is not None:
+            item["criteria"] = criteria
+        normalized[name] = item
+    return normalized
+
+
+def _get_context(arguments: Mapping[str, Any], context: ValidatorToolContext) -> str:
+    selected = arguments.get("context")
+    if selected == "spec":
+        return context.spec_text
+    if selected == "diff":
+        if context.diff_accessor is None:
+            raise ValueError("task diff is unavailable")
+        if context.diff_refs:
+            return str(context.diff_accessor(*context.diff_refs))
+        return str(context.diff_accessor())
+    if selected != "files":
+        raise ValueError("context must be diff, spec, or files")
+    paths = arguments.get("files")
+    if not isinstance(paths, list) or not 1 <= len(paths) <= _MAX_FILES:
+        raise ValueError(f"files context requires between 1 and {_MAX_FILES} paths")
+    root = context.workspace_root.resolve()
+    contents = []
+    for value in paths:
+        if not isinstance(value, str) or not value:
+            raise ValueError("file paths must be non-empty strings")
+        target = (root / value).resolve()
+        if not target.is_relative_to(root) or not target.is_file():
+            raise ValueError(f"file path is outside the workspace or unreadable: {value}")
+        contents.append(f"--- {value} ---\n{target.read_text(encoding='utf-8')}")
+    return "\n".join(contents)
+
+
+def _render(payload: Mapping[str, Any], questions: Mapping[str, Any], truncated: bool) -> str:
+    answers = payload["answers"]
+    lines = []
+    for name, question in questions.items():
+        answer = answers.get(name)
+        if not isinstance(answer, Mapping):
+            return "TypeSafe Jev error: malformed response; an answer is missing or invalid."
+        kind = question["type"]
+        if kind == "noul":
+            lines.append(f"{name} [noul]: {answer.get('noul')} (probability)")
+        elif kind == "choice":
+            lines.append(
+                f"{name} [choice]: {answer.get('choice')} (probabilities={_compact(answer.get('probabilities'))}; "
+                f"confidence={answer.get('confidence')})"
+            )
+        else:
+            lines.append(
+                f"{name} [score]: {answer.get('score')} (legend={_compact(answer.get('legend'))}; "
+                f"probabilities={_compact(answer.get('probabilities'))}; confidence={answer.get('confidence')})"
+            )
+    usage = payload.get("usage", {})
+    if not isinstance(usage, Mapping):
+        usage = {}
+    lines.append(f"usage: input_tokens={usage.get('input_tokens', usage.get('input', 'unknown'))}, "
+                 f"output_tokens={usage.get('output_tokens', usage.get('output', 'unknown'))}")
+    if truncated:
+        lines.append("context truncated to max_input_chars")
+    return "\n".join(lines)
+
+
+def _compact(value: Any) -> str:
+    return json.dumps(value, separators=(",", ":"), ensure_ascii=False)
+
+
+tool = ValidatorTool(
+    definition={
+        "type": "function",
+        "function": {
+            "name": "decide",
+            "description": "Ask TypeSafe Jev typed questions about the task context.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "context": {"type": "string", "enum": ["diff", "spec", "files"]},
+                    "files": {"type": "array", "items": {"type": "string"}, "maxItems": _MAX_FILES},
+                    "questions": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "name": {"type": "string"},
+                                "type": {"type": "string", "enum": ["noul", "choice", "score"]},
+                                "instructions": {"type": "string"},
+                                "criteria": {"type": ["object", "array", "null"]},
+                            },
+                            "required": ["name", "type", "instructions"],
+                            "additionalProperties": False,
+                        },
+                    },
+                },
+                "required": ["context", "questions"],
+                "additionalProperties": False,
+            },
+        },
+    },
+    handler=handle_decide,
+)

snodo_validator_tool_jev-0.23.3/src/snodo_validator_tool_jev.egg-info/PKG-INFO

@@ -0,0 +1,28 @@
+Metadata-Version: 2.4
+Name: snodo-validator-tool-jev
+Version: 0.23.3
+Summary: TypeSafe Jev decision tool for Snodo validators
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: snodo-engine==0.23.3
+Requires-Dist: httpx>=0.27.0
+
+# Snodo TypeSafe Jev validator tool
+
+An optional plugin that adds the `decide` function to opted-in LLM validators.
+Jev returns typed answers and calibrated probabilities; the LLM remains
+responsible for writing the validator verdict.
+
+Install it into the same Python environment as Snodo and configure
+`validator_tools: [{name: decide}]` on an LLM validator. The tool accepts
+`context` as `diff`, `spec`, or `files` (with up to a small number of repository
+relative file paths), and a list of named `questions` using `noul`, `choice`, or
+`score` types. Choice criteria map option names to descriptions; score criteria
+are ordered level labels.
+
+Per-tool protocol options are `model` (default `jev-latest`), `api_key_env`
+(default `TYPESAFE_API_KEY`), `base_url` (default `https://api.typesafe.ai`),
+`timeout_seconds`, and `max_input_chars`. The key is read only from the named
+environment variable. Each result line reports the typed answer and
+probabilities; usage and any context truncation follow. Jev pricing is $0.042
+per million input tokens; its maximum context length is unpublished.

snodo_validator_tool_jev-0.23.3/src/snodo_validator_tool_jev.egg-info/SOURCES.txt

@@ -0,0 +1,10 @@
+README.md
+pyproject.toml
+src/snodo_validator_tool_jev/__init__.py
+src/snodo_validator_tool_jev.egg-info/PKG-INFO
+src/snodo_validator_tool_jev.egg-info/SOURCES.txt
+src/snodo_validator_tool_jev.egg-info/dependency_links.txt
+src/snodo_validator_tool_jev.egg-info/entry_points.txt
+src/snodo_validator_tool_jev.egg-info/requires.txt
+src/snodo_validator_tool_jev.egg-info/top_level.txt
+tests/test_decide.py

snodo_validator_tool_jev-0.23.3/src/snodo_validator_tool_jev.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

snodo_validator_tool_jev-0.23.3/src/snodo_validator_tool_jev.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[snodo.validator_tools]
+decide = snodo_validator_tool_jev:tool

snodo_validator_tool_jev-0.23.3/src/snodo_validator_tool_jev.egg-info/requires.txt

@@ -0,0 +1,2 @@
+snodo-engine==0.23.3
+httpx>=0.27.0

snodo_validator_tool_jev-0.23.3/src/snodo_validator_tool_jev.egg-info/top_level.txt

@@ -0,0 +1 @@
+snodo_validator_tool_jev

snodo_validator_tool_jev-0.23.3/tests/test_decide.py

@@ -0,0 +1,164 @@
+from pathlib import Path
+from types import SimpleNamespace
+
+import httpx
+import pytest
+
+from snodo.validators.tool_registry import ValidatorToolContext, ValidatorToolRegistry
+from snodo_validator_tool_jev import tool
+
+
+@pytest.fixture
+def context(tmp_path):
+    return ValidatorToolContext(
+        workspace_root=tmp_path,
+        spec_text="task spec",
+        validator_id="review",
+        tool_options={"max_input_chars": 100},
+        diff_refs=("base", "head"),
+        diff_accessor=lambda *refs: f"diff {refs}",
+    )
+
+
+def test_discovery_request_and_render(monkeypatch, context):
+    plugin = SimpleNamespace(name="decide", load=lambda: tool)
+    monkeypatch.setattr("importlib.metadata.entry_points", lambda *, group: [plugin])
+    registry = ValidatorToolRegistry()
+    registry.discover_plugins()
+    registered = registry.lookup("decide")
+    assert registered is tool
+    assert registered.name == "decide"
+    monkeypatch.setenv("TYPESAFE_API_KEY", "secret-value")
+    captured = {}
+
+    class Response:
+        def raise_for_status(self):
+            return None
+
+        def json(self):
+            return {
+                "model": "jev-latest",
+                "answers": {
+                    "safe": {"noul": 0.9},
+                    "kind": {"choice": "good", "probabilities": {"good": 0.8}, "confidence": 0.8},
+                    "quality": {"score": "high", "legend": ["low", "high"], "probabilities": [0.1, 0.9], "confidence": 0.9},
+                },
+                "usage": {"input_tokens": 11, "output_tokens": 4},
+            }
+
+    def post(url, **kwargs):
+        captured.update(url=url, **kwargs)
+        return Response()
+
+    monkeypatch.setattr("snodo_validator_tool_jev.httpx.post", post)
+    args = {
+        "context": "diff",
+        "questions": [
+            {"name": "safe", "type": "noul", "instructions": "Is safe?"},
+            {"name": "kind", "type": "choice", "instructions": "Choose", "criteria": {"good": "Good", "bad": None}},
+            {"name": "quality", "type": "score", "instructions": "Rate", "criteria": ["low", "high"]},
+        ],
+    }
+    result = registered.handler(args, context)
+    request = captured["json"]
+    assert captured["url"] == "https://api.typesafe.ai/v1/systemone"
+    assert captured["headers"]["Authorization"] == "Bearer secret-value"
+    assert request["state"] == "diff ('base', 'head')"
+    assert request["questions"]["safe"] == {"type": "noul", "instructions": "Is safe?"}
+    assert request["questions"]["kind"]["criteria"] == {"good": "Good", "bad": None}
+    assert request["questions"]["quality"]["criteria"] == ["low", "high"]
+    assert "safe [noul]: 0.9" in result
+    assert "kind [choice]: good" in result and "confidence=0.8" in result
+    assert "quality [score]: high" in result and "probabilities=[0.1,0.9]" in result
+    assert "input_tokens=11" in result and "output_tokens=4" in result
+    assert "secret-value" not in result
+
+
+@pytest.mark.parametrize(
+    ("failure", "message"),
+    [
+        ("missing", "TYPESAFE_API_KEY is not set"),
+        ("http", "HTTP 503"),
+        ("timeout", "request timed out"),
+    ],
+)
+def test_readable_failures(monkeypatch, context, failure, message):
+    args = {"context": "spec", "questions": [{"name": "q", "type": "noul", "instructions": "Check"}]}
+    monkeypatch.delenv("TYPESAFE_API_KEY", raising=False)
+    if failure == "missing":
+        result = tool.handler(args, context)
+    else:
+        monkeypatch.setenv("TYPESAFE_API_KEY", "never-print-this")
+        if failure == "timeout":
+            def post(*args, **kwargs):
+                raise httpx.TimeoutException("never-print-this")
+        else:
+            class Response:
+                status_code = 503
+
+                def raise_for_status(self):
+                    request = httpx.Request("POST", "https://example.test")
+                    response = httpx.Response(503, request=request)
+                    raise httpx.HTTPStatusError("failure", request=request, response=response)
+
+            def post(*args, **kwargs):
+                return Response()
+        monkeypatch.setattr("snodo_validator_tool_jev.httpx.post", post)
+        result = tool.handler(args, context)
+    assert message in result
+    assert "never-print-this" not in result
+
+
+def test_truncates_context_and_confines_file_paths(monkeypatch, context, caplog):
+    monkeypatch.setenv("TYPESAFE_API_KEY", "secret-value")
+    context = ValidatorToolContext(context.workspace_root, "x" * 101, "review", {"max_input_chars": 10})
+    request_body = {}
+
+    class Response:
+        def raise_for_status(self):
+            return None
+
+        def json(self):
+            return {"answers": {"q": {"noul": 0.5}}, "usage": {}}
+
+    def post(url, **kwargs):
+        request_body.update(kwargs["json"])
+        return Response()
+
+    monkeypatch.setattr("snodo_validator_tool_jev.httpx.post", post)
+    args = {"context": "spec", "questions": [{"name": "q", "type": "noul", "instructions": "Check"}]}
+    result = tool.handler(args, context)
+    assert request_body["state"] == "x" * 10
+    assert "context truncated" in result
+    assert "secret-value" not in caplog.text
+
+    context = ValidatorToolContext(context.workspace_root, context.spec_text, "review", {"max_input_chars": 100})
+    path = context.workspace_root / "source.py"
+    path.write_text("safe", encoding="utf-8")
+    tool.handler(
+        {"context": "files", "files": ["source.py"], "questions": args["questions"]}, context
+    )
+    assert request_body["state"] == "--- source.py ---\nsafe"
+    escaped = Path("../outside.txt")
+    assert "ValueError" in tool.handler(
+        {"context": "files", "files": [str(escaped)], "questions": args["questions"]}, context
+    )
+
+
+def test_malformed_response_is_returned_as_tool_text(monkeypatch, context):
+    monkeypatch.setenv("TYPESAFE_API_KEY", "secret-value")
+
+    class Response:
+        def raise_for_status(self):
+            return None
+
+        def json(self):
+            return {"answers": []}
+
+    monkeypatch.setattr("snodo_validator_tool_jev.httpx.post", lambda *args, **kwargs: Response())
+    result = tool.handler(
+        {"context": "spec", "questions": [{"name": "q", "type": "noul", "instructions": "Check"}]},
+        context,
+    )
+    assert "malformed response" in result
+    assert "secret-value" not in result