evalkit-py 0.2.0.dev12__py3-none-any.whl

evalkit/cli/init_server.py

@@ -0,0 +1,165 @@
+import importlib.metadata
+import json
+import shutil
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+import typer
+import yaml
+from dotenv import dotenv_values
+
+from evalkit import version
+from evalkit.cli.client import ApiError
+from evalkit.paths import skills_root
+
+DEFAULT_DIR = Path("evalkit-try")
+ENCRYPTION_KEY_ENV = "EVALKIT_ENCRYPTION_KEY"
+DEFAULT_DATABASE_URL = "postgresql+psycopg://localhost/evalkit"
+DEFAULT_ADMIN_PASSWORD = "evalkit-admin"
+KIT_REQUIREMENT = '"evalkit-py[otel]"'
+KIT_LEAVES_OUT = ("*.whl", "__pycache__", "*.pyc", ".env")
+DOCKER_ONLY = ("docker-compose.yml", ".env.example")
+
+
+@dataclass(frozen=True)
+class Question:
+    key: str
+    env: str
+    prompt: str
+    default: str | None = None
+    secret: bool = False
+
+    @property
+    def flag(self) -> str:
+        return f"--{self.key.replace('_', '-')}"
+
+
+QUESTIONS = (
+    Question(
+        "database_url",
+        "EVALKIT_DATABASE_URL",
+        "Postgres URL: where evalkit stores users, cases, experiments and scores "
+        "(the database is created if missing)",
+        DEFAULT_DATABASE_URL,
+    ),
+    Question(
+        "llm_base_url",
+        "LLM_BASE_URL",
+        "LLM base URL: an OpenAI-compatible endpoint the judge and the sample agent call "
+        "(e.g. https://api.openai.com/v1)",
+    ),
+    Question(
+        "llm_model",
+        "LLM_MODEL",
+        "LLM model: the model name at that endpoint (e.g. gpt-4o-mini)",
+    ),
+    Question(
+        "llm_api_key",
+        "LLM_API_KEY",
+        "LLM API key: kept in .env, never in config.yaml",
+        secret=True,
+    ),
+    Question(
+        "admin_password",
+        "EVALKIT_ADMIN_PASSWORD",
+        (
+            "Admin password: you log in to the web UI and the CLI as `admin` with it; "
+            "admin can add users and workspaces and change any setting"
+        ),
+        DEFAULT_ADMIN_PASSWORD,
+        secret=True,
+    ),
+)
+
+DATABASE_NOTICE = (
+    "evalkit init-server creates the database named in the Postgres URL on that Postgres "
+    "server if it does not exist yet, and creates evalkit's tables in it."
+)
+
+
+@dataclass(frozen=True)
+class Answers:
+    database_url: str
+    llm_base_url: str
+    llm_model: str
+    llm_api_key: str
+    admin_password: str
+
+
+def resolve(flags: dict[str, str | None], *, yes: bool) -> Answers:
+    values = {}
+    for question in QUESTIONS:
+        given = flags.get(question.key)
+        if given is not None:
+            values[question.key] = given
+            continue
+        if yes:
+            if question.default is None:
+                raise ApiError(f"{question.flag} (or {question.env}) is required with --yes")
+            values[question.key] = question.default
+            continue
+        values[question.key] = typer.prompt(
+            question.prompt,
+            default=question.default,
+            hide_input=question.secret and sys.stdin.isatty(),
+        )
+    return Answers(**values)
+
+
+def kit_root() -> Path:
+    return skills_root() / "evalkit-deployment" / "try"
+
+
+def _direct_url() -> dict:
+    text = importlib.metadata.distribution(version.DISTRIBUTION).read_text("direct_url.json")
+    return json.loads(text) if text else {}
+
+
+def pinned_requirement() -> str:
+    direct = _direct_url()
+    if direct.get("dir_info", {}).get("editable"):
+        return f"evalkit-py[otel] @ {direct['url']}"
+    return f"evalkit-py[otel]=={version.current()}"
+
+
+def kit_files_in(target: Path) -> list[str]:
+    return sorted(path.name for path in kit_root().iterdir() if (target / path.name).exists())
+
+
+def copy_kit(target: Path, *, docker: bool) -> None:
+    left_out = KIT_LEAVES_OUT if docker else (*KIT_LEAVES_OUT, *DOCKER_ONLY)
+    shutil.copytree(
+        kit_root(), target, ignore=shutil.ignore_patterns(*left_out), dirs_exist_ok=True
+    )
+    run_py = target / "run.py"
+    run_py.write_text(run_py.read_text().replace(KIT_REQUIREMENT, f'"{pinned_requirement()}"'))
+
+
+def server_config(database_url: str, data_dir: Path) -> dict:
+    data = yaml.safe_load((kit_root() / "config.yaml").read_text())
+    data["db"] = {"url": database_url}
+    data["blob"] = {"url": data_dir.resolve().as_uri()}
+    return data
+
+
+def existing_encryption_key(target: Path) -> str | None:
+    return dotenv_values(target / ".env").get(ENCRYPTION_KEY_ENV)
+
+
+def write_server_files(target: Path, answers: Answers, encryption_key: str | None) -> None:
+    config = server_config(answers.database_url, target / "data")
+    (target / "config.yaml").write_text(yaml.safe_dump(config, sort_keys=False))
+    env = target / ".env"
+    env.write_text(
+        f"LLM_BASE_URL={answers.llm_base_url}\n"
+        f"LLM_MODEL={answers.llm_model}\n"
+        f"LLM_API_KEY={answers.llm_api_key}\n"
+        f"EVALKIT_ADMIN_PASSWORD={answers.admin_password}\n"
+        + (
+            f"{ENCRYPTION_KEY_ENV}={encryption_key}\n"
+            if encryption_key
+            else f"# {ENCRYPTION_KEY_ENV}=<the encryption_key this database was set up with>\n"
+        )
+    )
+    env.chmod(0o600)

evalkit/cli/local_run.py

@@ -0,0 +1,195 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+
+import httpx
+
+from evalkit import version
+from evalkit.client.agents import HttpAgents
+from evalkit.client.api import AsyncApi
+from evalkit.client.blob import HttpBlobStore
+from evalkit.client.connection import Connection
+from evalkit.client.evaluators import HttpEvaluators
+from evalkit.client.sessions import HttpSessionReader
+from evalkit.client.store import HttpStore, local_holder
+from evalkit.config import LocalConfig
+from evalkit.contracts.enums import ExperimentStatus
+from evalkit.contracts.settings import JudgeSettings
+from evalkit.core.agents import explicit
+from evalkit.core.harness_check import check_harness
+from evalkit.core.runner import (
+    ExperimentRequest,
+    RunnerDeps,
+    dropped_cases_text,
+    harness_capabilities,
+    missing_local_keys,
+    rescore_experiment,
+    run_experiment,
+)
+from evalkit.core.workspace import Reference, slug
+from evalkit.credentials import CredentialRegistry
+from evalkit.store.base import GitRevision, Selection
+
+
+def credentials(local: LocalConfig, conn: Connection) -> CredentialRegistry:
+    return CredentialRegistry(local.credentials, local.environ(), home=conn.username)
+
+
+def holder_for(conn: Connection) -> str:
+    return local_holder(conn.username or "unknown")
+
+
+def _deps(
+    local: LocalConfig,
+    conn: Connection,
+    api: AsyncApi,
+    exp_id: int,
+    progress: Callable[[str], None],
+    harness_transport: httpx.AsyncBaseTransport | None,
+    custom_eval_transport: httpx.AsyncBaseTransport | None,
+) -> RunnerDeps:
+    return RunnerDeps(
+        cfg=local,
+        store=HttpStore(api),
+        blob_store=HttpBlobStore(api, exp_id),
+        agents=HttpAgents(api),
+        evaluators=HttpEvaluators(api),
+        credentials=credentials(local, conn),
+        harness_transport=harness_transport,
+        custom_eval_transport=custom_eval_transport,
+        progress=progress,
+        sessions=HttpSessionReader(api),
+    )
+
+
+async def run_local(
+    conn: Connection,
+    local: LocalConfig,
+    *,
+    agent: Reference,
+    dataset_slug: str,
+    selection: Selection,
+    repetitions: int,
+    model: str | None,
+    resume: int | None,
+    revision: GitRevision,
+    progress: Callable[[str], None],
+    judge: JudgeSettings | None = None,
+    transport: httpx.AsyncBaseTransport | None = None,
+    custom_eval_transport: httpx.AsyncBaseTransport | None = None,
+    api_transport: httpx.AsyncBaseTransport | None = None,
+) -> int:
+    api = AsyncApi(conn.url, conn.token, transport=api_transport)
+    holder = holder_for(conn)
+    try:
+        await api.require_same_version()
+        store = HttpStore(api)
+        if resume is None:
+            definition = await HttpAgents(api).get(explicit(agent))
+            caps = await harness_capabilities(
+                definition, credentials(local, conn).key_for("agent", definition.ref), transport
+            )
+            exp, dropped = await store.start_experiment(
+                ExperimentRequest(
+                    dataset=dataset_slug,
+                    agent=slug(agent),
+                    selection=selection,
+                    repetitions=repetitions,
+                    model=model,
+                    capabilities=caps,
+                    cases=revision,
+                    evalkit_version=version.current(),
+                    judge=judge,
+                    holder=holder,
+                )
+            )
+            if dropped:
+                progress(dropped_cases_text(dropped))
+        else:
+            exp = await store.transition(resume, ExperimentStatus.RUNNING, holder)
+        progress(f"experiment {exp.id} (#{exp.number} of {exp.dataset})")
+        deps = _deps(local, conn, api, exp.id, progress, transport, custom_eval_transport)
+        for line in await missing_local_keys(
+            deps, await deps.agents.get(explicit(agent)), await store.pinned_cases(exp.id)
+        ):
+            progress(line)
+        await run_experiment(deps, exp.id, holder)
+        return exp.id
+    finally:
+        await api.aclose()
+
+
+async def evaluate_local(
+    conn: Connection,
+    local: LocalConfig,
+    exp_id: int,
+    *,
+    progress: Callable[[str], None],
+    custom_eval_transport: httpx.AsyncBaseTransport | None = None,
+    api_transport: httpx.AsyncBaseTransport | None = None,
+    refetch_trajectories: bool = False,
+) -> None:
+    api = AsyncApi(conn.url, conn.token, transport=api_transport)
+    holder = holder_for(conn)
+    try:
+        await api.require_same_version()
+        deps = _deps(local, conn, api, exp_id, progress, None, custom_eval_transport)
+        await deps.store.transition(exp_id, ExperimentStatus.EVALUATING, holder)
+        await rescore_experiment(deps, exp_id, holder, refetch_trajectories=refetch_trajectories)
+    finally:
+        await api.aclose()
+
+
+async def mark_interrupted(conn: Connection, exp_id: int, user: str) -> None:
+    api = AsyncApi(conn.url, conn.token)
+    try:
+        await HttpStore(api).transition(
+            exp_id,
+            ExperimentStatus.FAILED,
+            holder_for(conn),
+            error=f"interrupted: cancelled by {user}",
+        )
+    finally:
+        await api.aclose()
+
+
+async def local_input_schema(
+    conn: Connection,
+    local: LocalConfig,
+    agent: Reference,
+    *,
+    transport: httpx.AsyncBaseTransport | None = None,
+    api_transport: httpx.AsyncBaseTransport | None = None,
+) -> dict:
+    api = AsyncApi(conn.url, conn.token, transport=api_transport)
+    try:
+        definition = await HttpAgents(api).get(explicit(agent))
+    finally:
+        await api.aclose()
+    key = credentials(local, conn).key_for("agent", definition.ref)
+    caps = await harness_capabilities(definition, key, transport)
+    return caps.effective_input_schema
+
+
+async def check_local(
+    conn: Connection,
+    local: LocalConfig,
+    agent: Reference,
+    *,
+    transport: httpx.AsyncBaseTransport | None = None,
+    api_transport: httpx.AsyncBaseTransport | None = None,
+) -> dict[str, dict]:
+    api = AsyncApi(conn.url, conn.token, transport=api_transport)
+    try:
+        blob_store = HttpBlobStore(api, None)
+        definition = await HttpAgents(api).get(explicit(agent))
+        return await check_harness(
+            definition,
+            credentials(local, conn).key_for("agent", definition.ref),
+            blob_store,
+            transport=transport,
+            sessions=HttpSessionReader(api),
+            agent_slug=slug(agent),
+        )
+    finally:
+        await api.aclose()