valcore 0.2.4__tar.gz → 0.2.5__tar.gz

{valcore-0.2.4 → valcore-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: valcore
-Version: 0.2.4
+Version: 0.2.5
 Summary: A small, self-contained local tool for developing, improving, and running agentic evaluations.
 Author-email: Duncan McKinnon <dkmckinnon@outlook.com>
 License-Expression: Apache-2.0

{valcore-0.2.4 → valcore-0.2.5}/src/valcore/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.4'
-__version_tuple__ = version_tuple = (0, 2, 4)
+__version__ = version = '0.2.5'
+__version_tuple__ = version_tuple = (0, 2, 5)
 
 __commit_id__ = commit_id = None

valcore-0.2.5/src/valcore/agent_generator.py

@@ -0,0 +1,204 @@
+"""Draft agent versions from a natural-language description of the agent."""
+
+import re
+import string
+from typing import Any
+
+from pydantic import BaseModel
+from pydantic_ai import Agent as PydanticAgent
+
+from valcore.capabilities import spec_capability_names
+from valcore.errors import ConfigError
+from valcore.generator import ConfigGenerator, GeneratedConfigBase, Refinement
+from valcore.models import AgentVersion, FieldType, OutputField, validate_agent_version
+
+# A simple column placeholder is letters, digits, and underscores, with no
+# attribute access, indexing, conversion, or format specifier.
+_SIMPLE_COLUMN = re.compile(r"[A-Za-z0-9_]+")
+
+_JSON_TYPES: dict[FieldType, str] = {
+    FieldType.STR: "string",
+    FieldType.INT: "integer",
+    FieldType.FLOAT: "number",
+    FieldType.BOOL: "boolean",
+    FieldType.ENUM: "string",
+}
+
+
+def output_schema(fields: list[OutputField]) -> dict[str, Any]:
+    """Render output fields as the JSON Schema object an agent spec holds.
+
+    A repeated field name raises ConfigError rather than letting the JSON object silently
+    keep only the last field.
+    """
+    properties: dict[str, Any] = {}
+    for field in fields:
+        if field.name in properties:
+            raise ConfigError(f"Output field names must be unique; {field.name!r} repeats.")
+        prop: dict[str, Any] = {"type": _JSON_TYPES[field.type], "description": field.description}
+        if field.type is FieldType.ENUM:
+            prop["enum"] = field.enum_values
+        if field.minimum is not None:
+            prop["minimum"] = field.minimum
+        if field.maximum is not None:
+            prop["maximum"] = field.maximum
+        properties[field.name] = prop
+    return {
+        "type": "object",
+        "properties": properties,
+        "required": [field.name for field in fields if field.required],
+    }
+
+
+class GeneratedAgentConfig(GeneratedConfigBase):
+    """A complete agent version draft produced by the agent generator."""
+
+    output_fields: list[OutputField]
+    rationale: str
+
+    def spec(self) -> dict[str, Any]:
+        """Return the pydantic-ai spec blob this draft stores, omitting empty sections."""
+        spec: dict[str, Any] = {"instructions": self.instructions}
+        if self.capabilities:
+            spec["capabilities"] = [{c.name: c.config} for c in self.capabilities]
+        if self.output_fields:
+            spec["output_schema"] = output_schema(self.output_fields)
+        return spec
+
+    def to_version(self, model: str) -> AgentVersion:
+        """Return the unsaved agent version this draft describes."""
+        return AgentVersion(
+            agent_id="",
+            version_name=self.version_name,
+            model=model,
+            spec=self.spec(),
+            prompt_template=self.prompt_template,
+            required_columns=self.required_columns,
+            deps_mapping={},
+        )
+
+
+class AgentRefinement(Refinement[GeneratedAgentConfig]):
+    """Type the inherited refine flow for agents; no route uses it yet."""
+
+
+class AgentDraft(BaseModel):
+    """An agent version draft as the editor receives it; it carries no model."""
+
+    version_name: str
+    spec: dict[str, Any]
+    prompt_template: str
+    required_columns: list[str]
+    deps_mapping: dict[str, str]
+    rationale: str
+
+    @classmethod
+    def from_config(cls, config: GeneratedAgentConfig) -> "AgentDraft":
+        """Build the editor-facing draft from a validated generated config."""
+        return cls(
+            version_name=config.version_name,
+            spec=config.spec(),
+            prompt_template=config.prompt_template,
+            required_columns=config.required_columns,
+            deps_mapping={},
+            rationale=config.rationale,
+        )
+
+
+class AgentGenerator(ConfigGenerator[GeneratedAgentConfig]):
+    """Draft agent versions from a description of what the agent should do."""
+
+    subject = "agent"
+    request_label = "Agent description"
+    config_type = GeneratedAgentConfig
+    refinement_type = AgentRefinement
+
+    def role_instructions(self) -> str:
+        """Describe the generator's job and how to use ``rationale``."""
+        return (
+            "You design AI agents whose responses will be measured against datasets. "
+            "Given a natural-language description of what an agent should do, produce a "
+            "complete, valid agent version configuration.\n\n"
+            "Use `rationale` to briefly explain why you chose these instructions, inputs, "
+            "capabilities, and output shape."
+        )
+
+    def allowed_capabilities(self) -> list[str]:
+        """Return the harness capabilities an agent spec can reconstruct."""
+        return sorted(spec_capability_names())
+
+    def field_rules(self) -> str:
+        """Add the agent-specific rules to the shared structural requirements."""
+        return (
+            super().field_rules()
+            + "\n"
+            + (
+                "- `instructions` is the agent's system prompt: its role, behavior, and "
+                "constraints.\n"
+                "- `prompt_template` must contain at least one placeholder, and placeholders "
+                "are simple `{column}` names made of letters, digits, and underscores, with no "
+                "format specifiers.\n"
+                "- `required_columns` must list at least one column.\n"
+                "- `output_fields` is empty for an agent that answers in free text. Fill it "
+                "only when the description asks for structured output; field names must be "
+                "unique identifiers, an enum field needs `enum_values`, and `minimum`/"
+                "`maximum` apply only to int and float fields."
+            )
+        )
+
+    def check_version(self, version: AgentVersion) -> None:
+        """Reject a draft that breaks the agent rules or that the store would refuse."""
+        _check_agent_prompt_rules(version)
+        validate_agent_version(version)
+
+
+def _placeholders(template: str) -> list[tuple[str, str, str | None]]:
+    """Return each prompt-template ``{field}`` as ``(name, format_spec, conversion)``.
+
+    A template whose braces do not parse raises ConfigError so the generator's
+    single validation retry can ask for a corrected draft.
+    """
+    try:
+        # ValueError is raised while iterating, not when parse() is called.
+        parsed = list(string.Formatter().parse(template))
+    except ValueError as exc:
+        raise ConfigError(f"prompt_template has malformed placeholders: {exc}.") from exc
+    found: list[tuple[str, str, str | None]] = []
+    for _, field_name, format_spec, conversion in parsed:
+        if field_name:
+            found.append((field_name, format_spec or "", conversion))
+    return found
+
+
+def _check_agent_prompt_rules(version: AgentVersion) -> None:
+    """Enforce the prompt-template and required-column rules for a generated draft.
+
+    Store validation accepts a prompt with no placeholder, format specifiers such as
+    ``{question:>10}``, and an empty ``required_columns`` list. A generated draft must
+    bind to at least one dataset column through simple placeholders, so those are
+    rejected here before the store check.
+
+    Instructions are deliberately not checked: they are never formatted with row
+    columns, so any brace text in them (a JSON example, say) reaches the agent as-is.
+    """
+    prompt_fields = _placeholders(version.prompt_template)
+    if not prompt_fields:
+        raise ConfigError("prompt_template must contain at least one simple {column} placeholder.")
+    for name, format_spec, conversion in prompt_fields:
+        if conversion or format_spec or _SIMPLE_COLUMN.fullmatch(name) is None:
+            raise ConfigError(
+                "prompt_template placeholders must be simple {column} names made of "
+                "letters, digits, and underscores, with no conversion or format "
+                f"specifier; found field {name!r}, format_spec {format_spec!r}, "
+                f"conversion {conversion!r}."
+            )
+    if not version.required_columns:
+        raise ConfigError("required_columns must list at least one column.")
+
+
+async def generate_agent_draft(
+    prompt: str, *, model: str | None = None, agent: PydanticAgent | None = None
+) -> AgentDraft:
+    """Generate an editable agent version draft from a natural-language description."""
+    config = await AgentGenerator(model).generate(prompt, agent=agent)
+    return AgentDraft.from_config(config)

{valcore-0.2.4 → valcore-0.2.5}/src/valcore/api/deps.py

@@ -5,9 +5,10 @@ from typing import Annotated
 
 from fastapi import Depends
 
+from valcore import config
 from valcore.agent_prompt_sync import AgentPromptSync
 from valcore.logfire_prompt_variables import PromptVariableAdapter
-from valcore.settings import get_settings
+from valcore.settings import get_settings, is_local_cli_model
 from valcore.store import Store, create_engine, init_db
 
 
@@ -23,6 +24,18 @@ def get_store() -> Store:
     return Store(engine)
 
 
+def require_gateway_key_unless_local() -> None:
+    """Require the gateway key unless generation resolves to a local CLI model.
+
+    Generation routes pass no explicit model, so they resolve to
+    ``get_settings().default_model``. A ``local/<cli>`` default reaches an
+    already-logged-in CLI on this machine, never the gateway, so demanding a gateway
+    key there would block the exact keyless setup local CLI models exist for.
+    """
+    if not is_local_cli_model(get_settings().default_model):
+        config.require_gateway_key()
+
+
 def get_prompt_sync(store: Annotated[Store, Depends(get_store)]) -> AgentPromptSync:
     """Build the request's explicit prompt-sync service with the configured key."""
     adapter = PromptVariableAdapter()

{valcore-0.2.4 → valcore-0.2.5}/src/valcore/api/routes/agents.py

@@ -13,10 +13,11 @@ from fastapi import APIRouter, Depends
 from pydantic import BaseModel, ConfigDict, Field, ValidationError
 from starlette.concurrency import run_in_threadpool
 
-from valcore import agent_spec, config
+from valcore import agent_generator, agent_spec, config
+from valcore.agent_generator import AgentDraft
 from valcore.agent_prompt_sync import AgentPromptSync, SyncStatus, TemplateStatus
 from valcore.agent_spec import AgentSpec
-from valcore.api.deps import get_prompt_sync, get_store
+from valcore.api.deps import get_prompt_sync, get_store, require_gateway_key_unless_local
 from valcore.errors import ContractError, NotFoundError
 from valcore.factory import build_agent_from_version, execute_agent_version
 from valcore.models import Agent, AgentVersion, DatasetDerivation, DerivationState
@@ -41,6 +42,15 @@ class AgentCreate(BaseModel):
     description: str = ""
 
 
+class AgentGenerateRequest(BaseModel):
+    """The free-text description a new agent's first version is drafted from.
+
+    It steers generation only and is never stored.
+    """
+
+    prompt: str
+
+
 class AgentUpdate(BaseModel):
     """Partial mutable metadata for an agent."""
 
@@ -477,6 +487,17 @@ async def import_spec(body: AgentSpecImportRequest) -> AgentSpecImport:
         raise ContractError(f"Invalid valcore binding metadata: {exc}") from exc
 
 
+@router.post("/generate", response_model=AgentDraft)
+async def generate_agent(body: AgentGenerateRequest) -> AgentDraft:
+    """Draft an agent version from a description without persisting anything.
+
+    Used by the 'new agent' flow before any version exists. Calls an LLM and can take
+    tens of seconds; the UI fills an unsaved draft editor from the result.
+    """
+    require_gateway_key_unless_local()
+    return await agent_generator.generate_agent_draft(body.prompt)
+
+
 @router.post("/versions/{vid}/trial", response_model=TrialResult)
 async def trial_version(vid: str, body: TrialRequest, store: StoreDep) -> TrialResult:
     """Run a version once and return either its output or its model failure."""

{valcore-0.2.4 → valcore-0.2.5}/src/valcore/api/routes/datasets.py

@@ -10,8 +10,7 @@ from typing import Annotated, Literal
 from fastapi import APIRouter, Depends, File, Form, UploadFile
 from pydantic import BaseModel, ConfigDict, ValidationError
 
-from valcore import config
-from valcore.api.deps import get_store
+from valcore.api.deps import get_store, require_gateway_key_unless_local
 from valcore.config_io import EvalPackage
 from valcore.datagen import generate_rows
 from valcore.errors import ContractError
@@ -27,7 +26,6 @@ from valcore.models import (
     label_set_fields_from_schema,
 )
 from valcore.seeding import dataset_shape_from_version
-from valcore.settings import get_settings, is_local_cli_model
 from valcore.store import Store
 
 router = APIRouter(prefix="/api/datasets", tags=["datasets"])
@@ -39,18 +37,6 @@ _MAX_GENERATE_COUNT = 200
 _JSONL_INFER_LIMIT = 50
 
 
-def _require_gateway_key_unless_local() -> None:
-    """Require the gateway key unless generation resolves to a local CLI model.
-
-    None of the generate routes pass an explicit model, so they all resolve to
-    ``get_settings().default_model`` inside ``generate_rows``. A ``local/<cli>``
-    default reaches an already-logged-in CLI on this machine, never the gateway, so
-    demanding a gateway key there would block the exact keyless setup local models exist for.
-    """
-    if not is_local_cli_model(get_settings().default_model):
-        config.require_gateway_key()
-
-
 class DatasetCreate(BaseModel):
     """Request body to create an empty dataset."""
 
@@ -559,7 +545,7 @@ def _check_column_notes(column_notes: dict[str, str] | None, columns: list[str])
 @router.post("/generate")
 async def generate_dataset(body: DatasetGenerate, store: StoreDep) -> DatasetCreatedOut:
     """Generate a dataset and its rows with suggested labels."""
-    _require_gateway_key_unless_local()
+    require_gateway_key_unless_local()
     if body.count > _MAX_GENERATE_COUNT:
         raise ContractError(f"count may not exceed {_MAX_GENERATE_COUNT}.")
 
@@ -616,7 +602,7 @@ async def generate_dataset_from_version(
     body: DatasetGenerateFromVersion, store: StoreDep
 ) -> DatasetCreatedOut:
     """Generate a dataset shaped by an evaluator version, runnable against it by construction."""
-    _require_gateway_key_unless_local()
+    require_gateway_key_unless_local()
     if body.count > _MAX_GENERATE_COUNT:
         raise ContractError(f"count may not exceed {_MAX_GENERATE_COUNT}.")
 
@@ -903,7 +889,7 @@ async def generate_more_rows(id: str, body: RowsGenerate, store: StoreDep) -> li
     the new rows stay compatible with the existing ones and with any evaluator that already
     runs against them.
     """
-    _require_gateway_key_unless_local()
+    require_gateway_key_unless_local()
     if body.count > _MAX_GENERATE_COUNT:
         raise ContractError(f"count may not exceed {_MAX_GENERATE_COUNT}.")
 

{valcore-0.2.4 → valcore-0.2.5}/src/valcore/api/routes/evaluators.py

@@ -7,15 +7,14 @@ from typing import Annotated
 from fastapi import APIRouter, Depends
 from pydantic import BaseModel, ConfigDict
 
-from valcore import agent_spec, config, generator
-from valcore.api.deps import get_store
+from valcore import agent_spec, eval_generator
+from valcore.api.deps import get_store, require_gateway_key_unless_local
 from valcore.config_io import EvalPackage
 from valcore.errors import ContractError, FrozenVersionError
+from valcore.eval_generator import GeneratedConfig, RefinedConfig
 from valcore.export import render_judge_module, render_script
-from valcore.generator import GeneratedConfig, RefinedConfig
 from valcore.models import CapabilitySpec, Evaluator, LabelSchema, OutputField, ScoreKind
 from valcore.seeding import evaluator_seed_from_dataset
-from valcore.settings import get_settings, is_local_cli_model
 from valcore.store import Store
 
 router = APIRouter(prefix="/api/evaluators", tags=["evaluators"])
@@ -23,18 +22,6 @@ router = APIRouter(prefix="/api/evaluators", tags=["evaluators"])
 StoreDep = Annotated[Store, Depends(get_store)]
 
 
-def _require_gateway_key_unless_local() -> None:
-    """Require the gateway key unless generation resolves to a local CLI model.
-
-    None of the generate/refine routes pass an explicit model, so they all resolve to
-    ``get_settings().default_model`` inside ``generator``. A ``local/<cli>`` default
-    reaches an already-logged-in CLI on this machine, never the gateway, so demanding a
-    gateway key there would block the exact keyless setup local CLI models exist for.
-    """
-    if not is_local_cli_model(get_settings().default_model):
-        config.require_gateway_key()
-
-
 # -- Request bodies -----------------------------------------------------------
 
 
@@ -491,9 +478,9 @@ async def generate(body: GenerateRequest, store: StoreDep) -> GeneratedConfig:
     Calls an LLM and can take tens of seconds; the UI presents the returned config as an
     editable draft saved as a version separately.
     """
-    _require_gateway_key_unless_local()
+    require_gateway_key_unless_local()
     columns, label_schema, column_notes = _resolve_seed(body, store)
-    return await generator.generate_config(
+    return await eval_generator.generate_config(
         body.criteria,
         columns=columns,
         column_notes=column_notes,
@@ -509,10 +496,10 @@ async def generate_version(id: str, body: GenerateRequest, store: StoreDep) -> G
     of seconds; the UI presents the returned config as an editable draft that the user
     saves as a version separately.
     """
-    _require_gateway_key_unless_local()
+    require_gateway_key_unless_local()
     store.get_evaluator(id)
     columns, label_schema, column_notes = _resolve_seed(body, store)
-    return await generator.generate_config(
+    return await eval_generator.generate_config(
         body.criteria,
         columns=columns,
         column_notes=column_notes,
@@ -527,5 +514,5 @@ async def refine_version(body: RefineRequest) -> RefinedConfig:
     Calls an LLM and can take tens of seconds. The response includes ``changed_fields``
     for the diff view; nothing is saved until the user submits a new version.
     """
-    _require_gateway_key_unless_local()
-    return await generator.refine_config(body.config, body.instruction)
+    require_gateway_key_unless_local()
+    return await eval_generator.refine_config(body.config, body.instruction)

valcore-0.2.5/src/valcore/eval_generator.py

@@ -0,0 +1,218 @@
+"""Generate and refine evaluator configurations from natural language."""
+
+from pydantic_ai import Agent as PydanticAgent
+
+from valcore.errors import ConfigError
+from valcore.generator import ConfigGenerator, GeneratedConfigBase, Refinement
+from valcore.models import (
+    VALID_CAPABILITIES,
+    EvaluatorVersion,
+    LabelSchema,
+    OutputField,
+    ScoreKind,
+    validate_version,
+)
+from valcore.tools import tool_names
+
+
+class GeneratedConfig(GeneratedConfigBase):
+    """Complete evaluator version config produced by the generator."""
+
+    name: str
+    output_fields: list[OutputField]
+    score_field: str
+    score_kind: ScoreKind
+    score_labels: list[str] | None = None
+    score_minimum: float | None = None
+    score_maximum: float | None = None
+    tools: list[str]
+    rationale: str
+
+    def to_version(self, model: str) -> EvaluatorVersion:
+        """Build the unsaved evaluator version used for structural validation."""
+        return EvaluatorVersion(
+            evaluator_id="",
+            version_name=self.version_name,
+            model=model,
+            instructions=self.instructions,
+            prompt_template=self.prompt_template,
+            required_columns=self.required_columns,
+            output_fields=[f.model_dump() for f in self.output_fields],
+            score_field=self.score_field,
+            score_kind=self.score_kind,
+            score_labels=self.score_labels,
+            score_minimum=self.score_minimum,
+            score_maximum=self.score_maximum,
+            capabilities=[c.model_dump() for c in self.capabilities],
+            tools=self.tools,
+        )
+
+
+class RefinedConfig(Refinement[GeneratedConfig]):
+    """A full evaluator config plus a description of what a refinement changed."""
+
+
+class EvaluatorGenerator(ConfigGenerator[GeneratedConfig]):
+    """Drafts LLM-as-judge evaluator configs, optionally shaped by a dataset."""
+
+    subject = "evaluator"
+    request_label = "Criteria"
+    config_type = GeneratedConfig
+    refinement_type = RefinedConfig
+
+    def __init__(
+        self,
+        model: str | None = None,
+        *,
+        columns: list[str] | None = None,
+        column_notes: dict[str, str] | None = None,
+        label_schema: LabelSchema | None = None,
+    ) -> None:
+        """Keep optional dataset shape and score constraints for the draft prompt."""
+        super().__init__(model)
+        self.columns = columns
+        self.column_notes = column_notes
+        self.label_schema = label_schema
+
+    def role_instructions(self) -> str:
+        """Describe the evaluator designer's task to the generator agent."""
+        return (
+            "You design LLM-as-judge evaluators. Given natural-language criteria, "
+            "produce a complete, valid evaluator configuration.\n\n"
+            "Use `rationale` to briefly explain why you chose this schema, scoring, "
+            "capabilities, and tools."
+        )
+
+    def allowed_capabilities(self) -> list[str]:
+        """Return the capability names valid for evaluator versions."""
+        return sorted(VALID_CAPABILITIES)
+
+    def field_rules(self) -> str:
+        """Add evaluator score, output, and tool constraints to the shared rules."""
+        return (
+            super().field_rules()
+            + "\n"
+            + (
+                "- `instructions` is the system prompt given to the judge.\n"
+                "- `output_fields` is an ordered list of at least one field; field names "
+                "must be unique.\n"
+                "- `score_field` must name one of the `output_fields`.\n"
+                "- A categorical score requires `score_kind` 'categorical' and a score "
+                "field of type enum whose `enum_values` exactly equal `score_labels`. A "
+                "numeric score requires `score_kind` 'numeric', a score field typed int or "
+                "float, and `score_labels` null.\n"
+                "- `required_columns` must list at least one column: every column the "
+                "judge needs.\n"
+                f"- `tools` may use only these names: {tool_names()}. Never invent a tool "
+                "outside this list."
+            )
+        )
+
+    def check_version(self, version: EvaluatorVersion) -> None:
+        """Reject unknown tools, then apply the stored evaluator version rules."""
+        valid = sorted(tool_names())
+        unknown = sorted(set(version.tools) - set(valid))
+        if unknown:
+            raise ConfigError(f"Unknown tools {unknown}; valid names are {valid}.")
+        validate_version(version)
+
+    def generation_prompt(self, request: str) -> str:
+        """Append any supplied dataset columns and score-space guidance."""
+        return (
+            super().generation_prompt(request)
+            + _columns_section(self.columns, self.column_notes)
+            + _score_space_section(self.label_schema)
+        )
+
+
+def _columns_section(columns: list[str] | None, column_notes: dict[str, str] | None) -> str:
+    """Render the available-columns block, annotating each column's role when given.
+
+    Roles steer both `required_columns` and the `{column}` placeholders, so the
+    full column set (bare `columns` plus any keys in `column_notes`) is listed;
+    an annotated column that is context-only or ignored can then be excluded by
+    the model from both.
+    """
+    if not columns and not column_notes:
+        return ""
+    constraints = (
+        "\n\nColumn constraints:\n"
+        "  - `required_columns` may contain only the dataset columns listed above.\n"
+        "  - A column described as irrelevant or ignored must appear in neither "
+        "`required_columns` nor `prompt_template`.\n"
+        "  - Every `{column}` placeholder in `prompt_template` must name a listed dataset "
+        "column; do not invent columns or placeholders."
+    )
+    if not column_notes:
+        # Preserve the pre-existing plain listing when no roles are supplied.
+        return f"\n\nAvailable dataset columns: {columns}{constraints}"
+    # Union without losing the caller's ordering: listed columns first, then any
+    # annotated column not already named in `columns`.
+    ordered = list(columns or [])
+    for name in column_notes:
+        if name not in ordered:
+            ordered.append(name)
+    lines = [
+        f"  - {name}: {column_notes[name]}" if name in column_notes else f"  - {name}"
+        for name in ordered
+    ]
+    return (
+        "\n\nAvailable dataset columns and how each factors into the assessment:\n"
+        + "\n".join(lines)
+        + constraints
+    )
+
+
+def _score_space_section(label_schema: LabelSchema | None) -> str:
+    """Render the score-space constraint the config must honour, if one is given.
+
+    Kept as prompt text only: the constraint is stated, never enforced by
+    post-processing the agent's output.
+    """
+    if label_schema is None:
+        return ""
+    if label_schema.kind is ScoreKind.CATEGORICAL:
+        return (
+            "\n\nScore-space constraint (the generated config must honour it exactly):\n"
+            "  - `score_kind` must be categorical.\n"
+            f"  - `score_labels` must equal these labels exactly: {label_schema.labels}."
+        )
+    return (
+        "\n\nScore-space constraint (the generated config must honour it exactly):\n"
+        "  - `score_kind` must be numeric.\n"
+        f"  - the score bounds must match: minimum {label_schema.minimum}, "
+        f"maximum {label_schema.maximum}."
+    )
+
+
+async def generate_config(
+    criteria: str,
+    *,
+    columns: list[str] | None = None,
+    column_notes: dict[str, str] | None = None,
+    label_schema: LabelSchema | None = None,
+    model: str | None = None,
+    agent: PydanticAgent | None = None,
+) -> GeneratedConfig:
+    """Generate a complete evaluator config from natural-language criteria.
+
+    `column_notes` steers which columns become `required_columns` and which
+    `{column}` placeholders appear; `label_schema` states the required score
+    space as a constraint. Both are prompt steering only — the validate + single
+    retry contract is unchanged.
+    """
+    generator = EvaluatorGenerator(
+        model, columns=columns, column_notes=column_notes, label_schema=label_schema
+    )
+    return await generator.generate(criteria, agent=agent)
+
+
+async def refine_config(
+    current: GeneratedConfig,
+    instruction: str,
+    *,
+    model: str | None = None,
+    agent: PydanticAgent | None = None,
+) -> RefinedConfig:
+    """Apply a natural-language change request to an existing evaluator config."""
+    return await EvaluatorGenerator(model).refine(current, instruction, agent=agent)