verdikt-sdk 0.1.0__tar.gz

verdikt_sdk-0.1.0/.github/workflows/pre-commit.yml

@@ -0,0 +1,30 @@
+name: pre-commit
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Set up Python
+        run: uv python install 3.13
+
+      - name: Install dependencies
+        run: uv sync --group dev
+
+      - name: Run ruff lint
+        run: uv run ruff check verdikt_sdk/
+
+      - name: Run ruff format check
+        run: uv run ruff format --check verdikt_sdk/
+
+      - name: Run mypy
+        run: uv run mypy verdikt_sdk/

verdikt_sdk-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,86 @@
+name: publish
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Set up Python
+        run: uv python install 3.13
+
+      - name: Verify tag matches pyproject.toml version
+        run: |
+          PKG_VERSION=$(python3 -c "
+          import tomllib
+          with open('pyproject.toml', 'rb') as f:
+              print(tomllib.load(f)['project']['version'])
+          ")
+          TAG_VERSION="${GITHUB_REF_NAME#v}"
+          if [ "$PKG_VERSION" != "$TAG_VERSION" ]; then
+            echo "Tag version ($TAG_VERSION) does not match pyproject.toml version ($PKG_VERSION)"
+            exit 1
+          fi
+
+      - name: Build distribution
+        run: uv build
+
+      - name: Store distribution packages
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+  publish-to-pypi:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/verdikt-sdk
+
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download distribution packages
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: Create GitHub Release
+    needs: publish-to-pypi
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download distribution packages
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+      - name: Create GitHub Release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh release create "$GITHUB_REF_NAME" dist/* --generate-notes

verdikt_sdk-0.1.0/.gitignore

@@ -0,0 +1,163 @@
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/

verdikt_sdk-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,28 @@
+repos:
+- repo: local
+  hooks:
+  - id: ruff-lint
+    name: lint
+    language: system
+    types:
+      - python
+    entry: make ruff-lint
+    pass_filenames: false
+
+  - id: ruff-format
+    name: format
+    language: system
+    types:
+      - python
+    entry: make ruff-format
+    pass_filenames: false
+
+  - id: mypy
+    name: type-check
+    language: system
+    types:
+      - python
+    entry: make mypy
+    pass_filenames: false
+
+

verdikt_sdk-0.1.0/.python-version

@@ -0,0 +1 @@
+3.13

verdikt_sdk-0.1.0/Makefile

@@ -0,0 +1,21 @@
+.PHONY: help
+help: # Show help for each of the Makefile recipes
+	@grep -E '^[a-zA-Z0-9 -]+:.*#'  Makefile | sort | while read -r l; do printf "\033[1;32m$$(echo $$l | cut -f 1 -d':')\033[00m: $$(echo $$l | cut -f 2- -d'#')\n"; done
+
+TA ?= -v tests/
+
+ruff-lint: # Run ruff linter
+	uv run ruff check --fix verdikt_sdk/
+
+ruff-format: # Run ruff formatter
+	uv run ruff format verdikt_sdk/
+
+mypy: # Run mypy type checker
+	uv run mypy verdikt_sdk/
+
+lint: # Run pre-commit
+	pre-commit run --all-files
+
+test: # Run tests
+	uv run pytest $(TA)
+

verdikt_sdk-0.1.0/PKG-INFO

@@ -0,0 +1,56 @@
+Metadata-Version: 2.4
+Name: verdikt-sdk
+Version: 0.1.0
+Summary: Python SDK for the Verdikt Evaluation API
+Requires-Python: >=3.13
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: pydantic>=2.0
+Requires-Dist: yalc>=0.2.1
+Description-Content-Type: text/markdown
+
+# verdikt-sdk
+
+Python SDK for [Verdikt](https://github.com/cognitai-labs-dev/verdikt) — a standalone AI evaluation service that decouples evaluation and LLM/human judging from the application being evaluated.
+
+## Installation
+
+```
+pip install verdikt-sdk
+```
+
+## Usage
+
+```python
+from verdikt_sdk import EvaluationClient
+from verdikt_sdk.models import EvaluationType, Question
+from yalc import LLMModel
+
+client = EvaluationClient(
+    base_url="https://your-verdikt-instance.com",
+    client_id="your-client-id",
+    client_secret="your-client-secret",
+)
+
+# Register your app (idempotent — safe to call on every deploy)
+await client.create_app(slug="my-app", name="My App")
+
+# Sync questions to the dataset (idempotent)
+await client.add_questions("my-app", [
+    Question(question="What is the capital of France?", human_answer="Paris"),
+])
+
+# Run an evaluation cycle
+await client.run_evaluation(
+    app_slug="my-app",
+    app_version="v1.2.0",
+    callback=my_llm_function,  # async fn(question: str) -> str
+    evaluation_type=EvaluationType.LLM_ONLY,
+    llm_judge_models=[LLMModel.gpt_4o_mini],
+)
+```
+
+`run_evaluation` calls your `callback` concurrently for every question in the dataset, then submits all answers to Verdikt for judgment.
+
+## Authentication
+
+The SDK authenticates via Zitadel OAuth2 client credentials. Create a machine user in your Zitadel project and pass its `client_id` and `client_secret` to `EvaluationClient`.

verdikt_sdk-0.1.0/README.md

@@ -0,0 +1,46 @@
+# verdikt-sdk
+
+Python SDK for [Verdikt](https://github.com/cognitai-labs-dev/verdikt) — a standalone AI evaluation service that decouples evaluation and LLM/human judging from the application being evaluated.
+
+## Installation
+
+```
+pip install verdikt-sdk
+```
+
+## Usage
+
+```python
+from verdikt_sdk import EvaluationClient
+from verdikt_sdk.models import EvaluationType, Question
+from yalc import LLMModel
+
+client = EvaluationClient(
+    base_url="https://your-verdikt-instance.com",
+    client_id="your-client-id",
+    client_secret="your-client-secret",
+)
+
+# Register your app (idempotent — safe to call on every deploy)
+await client.create_app(slug="my-app", name="My App")
+
+# Sync questions to the dataset (idempotent)
+await client.add_questions("my-app", [
+    Question(question="What is the capital of France?", human_answer="Paris"),
+])
+
+# Run an evaluation cycle
+await client.run_evaluation(
+    app_slug="my-app",
+    app_version="v1.2.0",
+    callback=my_llm_function,  # async fn(question: str) -> str
+    evaluation_type=EvaluationType.LLM_ONLY,
+    llm_judge_models=[LLMModel.gpt_4o_mini],
+)
+```
+
+`run_evaluation` calls your `callback` concurrently for every question in the dataset, then submits all answers to Verdikt for judgment.
+
+## Authentication
+
+The SDK authenticates via Zitadel OAuth2 client credentials. Create a machine user in your Zitadel project and pass its `client_id` and `client_secret` to `EvaluationClient`.

verdikt_sdk-0.1.0/SDK.md

@@ -0,0 +1,177 @@
+# Evaluation SDK Spec
+
+Python SDK that wraps the evaluation API so integrators only provide a callback — the SDK handles auth, dataset diffing, and evaluation submission.
+
+---
+
+## Backend changes required (this repo)
+
+Four additions needed before the SDK can be built:
+
+### 1. Add `slug` to apps
+
+- Add a `slug` column to the `apps` table — unique, not null, URL-safe (lowercase, hyphens)
+- Enforced at the DB level with a unique constraint
+- `POST /v1/app` accepts `slug` alongside `name`
+- New endpoint: `GET /v1/app/by-slug/{slug}` → returns `AppSchema` (404 if not found)
+
+This replaces the need to fetch all apps and filter client-side.
+
+### 2. `GET /.well-known`
+Returns the Zitadel issuer URL so the SDK can discover it from `base_url` alone.
+
+```json
+{ "issuer": "https://my-zitadel.example.com" }
+```
+
+### 3. `GET /v1/app/{app_id}/datasets/hashes`
+Lightweight endpoint for SDK diffing — returns hashes only, no full text.
+
+```json
+[
+  { "id": 1, "question_hash": "sha256...", "human_answer_hash": "sha256..." },
+  { "id": 2, "question_hash": "sha256...", "human_answer_hash": "sha256..." }
+]
+```
+
+Hash algorithm: SHA-256 of the stripped text.
+
+### 4. `PATCH /v1/app/{app_id}/datasets/{dataset_id}`
+Updates `human_answer` (and optionally `question`) on an existing dataset entry.
+`AppDatasetUpdateSchema` already exists in `src/schemas/app_dataset.py` — just needs a route.
+
+---
+
+## SDK interface
+
+```python
+from eval_sdk import EvaluationClient
+from typing import Callable, Literal
+
+class EvaluationClient:
+    def __init__(
+        self,
+        base_url: str,       # e.g. "https://eval.mycompany.com"
+        client_id: str,      # Zitadel machine user client ID
+        client_secret: str,  # Zitadel machine user client secret
+    ) -> None: ...
+
+    def create_app(self, slug: str, name: str) -> None: ...
+
+    def add_questions(
+        self,
+        app_slug: str,
+        questions: list[dict],  # [{"question": str, "human_answer": str}]
+    ) -> None: ...
+
+    def run_evaluation(
+        self,
+        app_slug: str,
+        app_version: str,
+        callback: Callable[[str], str],
+        evaluation_type: Literal["LLM_ONLY", "HUMAN_AND_LLM"] = "LLM_ONLY",
+        llm_judge_models: list[str] | None = None,
+    ) -> None: ...
+```
+
+---
+
+## Method details
+
+### `create_app(slug, name)`
+Idempotent — safe to call on every deploy.
+
+1. `GET /v1/app/by-slug/{slug}` → if 200, app exists → no-op
+2. If 404 → `POST /v1/app` with `{ "slug": slug, "name": name }`
+
+### `add_questions(app_slug, questions)`
+Idempotent — safe to call on every deploy. Uses SHA-256 of the question text as the match key so full text is never compared directly (questions can be long).
+
+1. Resolve `app_slug` → `app_id` via `GET /v1/app/by-slug/{slug}` (cached per client instance)
+2. `GET /v1/app/{id}/datasets/hashes` → existing hashes
+3. For each incoming question, compute `sha256(question.strip())`:
+   - Hash **not found** → `POST /v1/app/{id}/datasets` (new question)
+   - Hash found, `human_answer_hash` **differs** → `PATCH /v1/app/{id}/datasets/{dataset_id}` (updated answer)
+   - Hash found, `human_answer_hash` **matches** → skip
+
+### `run_evaluation(app_slug, app_version, callback, ...)`
+1. Resolve `app_slug` → `app_id` via `GET /v1/app/by-slug/{slug}` (cached per client instance)
+2. `GET /v1/app/{id}/datasets` → full question list
+3. For each dataset item: `answer = callback(item["question"])`
+4. `POST /v1/app/{id}/evaluation` with:
+   ```json
+   {
+     "app_version": "<app_version>",
+     "evaluation_type": "<evaluation_type>",
+     "app_answers": { "<dataset_id>": "<answer>", ... },
+     "llm_judge_models": ["gpt-4o-mini"]
+   }
+   ```
+
+---
+
+## Auth
+
+Uses **OAuth2 client credentials grant** against Zitadel.
+
+Flow on first API call:
+1. `GET {base_url}/.well-known` → get `issuer`
+2. `POST {issuer}/oauth/v2/token` with `grant_type=client_credentials`, `client_id`, `client_secret`
+3. Cache the token; refresh automatically when `expires_in` is reached
+
+The `issuer` and token are cached on the client instance — no repeated discovery calls.
+
+---
+
+## Slug → ID caching
+
+All three methods resolve `app_slug` → `app_id` via `GET /v1/app/by-slug/{slug}`. The resolved mapping is cached on the client instance so multiple method calls don't repeat the lookup.
+
+---
+
+## Slug format
+
+- Lowercase, alphanumeric, hyphens only — e.g. `"my-app"`, `"gpt-wrapper-v2"`
+- Enforced by the API (422 if invalid format)
+- Chosen by the integrator at `create_app` time; stable forever
+
+---
+
+## Dependencies
+
+- `httpx` — HTTP client
+- `pydantic` — response validation
+
+---
+
+## Usage example
+
+```python
+from eval_sdk import EvaluationClient
+
+client = EvaluationClient(
+    base_url="https://eval.mycompany.com",
+    client_id="my-service@myproject.zitadel.cloud",
+    client_secret="...",
+)
+
+# Idempotent setup — safe to call on every deploy
+client.create_app(slug="my-app", name="My App")
+
+client.add_questions("my-app", [
+    {"question": "What is the capital of France?", "human_answer": "Paris"},
+    {"question": "What is 2 + 2?", "human_answer": "4"},
+])
+
+# Run after each inference cycle
+def my_llm(question: str) -> str:
+    return my_model.complete(question)
+
+client.run_evaluation(
+    app_slug="my-app",
+    app_version="v1.4.2",
+    callback=my_llm,
+    evaluation_type="LLM_ONLY",
+    llm_judge_models=["gpt-4o-mini"],
+)
+```