testrelic-deepeval 0.1.0__tar.gz

testrelic_deepeval-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,36 @@
+name: CI
+
+on:
+  push:
+    branches: [main, stage, prod]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ["3.9", "3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+          cache: pip
+
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Lint (ruff)
+        run: ruff check src tests
+
+      - name: Type check (mypy)
+        run: mypy src/testrelic
+
+      - name: Test (pytest)
+        run: pytest --cov=testrelic --cov-report=term-missing

testrelic_deepeval-0.1.0/.github/workflows/deepeval-compat.yml

@@ -0,0 +1,49 @@
+name: DeepEval compatibility check
+
+on:
+  schedule:
+    - cron: "0 6 * * 1"  # Weekly, Monday 06:00 UTC
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  bump-and-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Detect latest deepeval release and bump pin
+        run: |
+          python -m pip install --upgrade pip
+          latest=$(pip index versions deepeval 2>/dev/null \
+            | head -n1 \
+            | sed -E 's/.*\(([^)]+)\).*/\1/' || true)
+          echo "Latest deepeval: ${latest:-unknown}"
+          if [ -n "$latest" ]; then
+            python -c "import re, pathlib; p=pathlib.Path('pyproject.toml'); t=p.read_text(); p.write_text(re.sub(r'\"deepeval>=[^\"]+\"', f'\"deepeval>=${latest}\"', t))"
+          fi
+
+      - name: Install
+        run: pip install -e ".[dev]"
+
+      - name: Run translator tests
+        run: pytest tests/test_translators.py tests/test_capture.py -v
+
+      - name: Open PR on diff
+        uses: peter-evans/create-pull-request@v6
+        with:
+          branch: deepeval-compat/bump
+          title: "chore(deps): bump deepeval to latest"
+          commit-message: "chore(deps): bump deepeval pin"
+          body: |
+            Automated weekly compatibility sweep.
+            Translator tests passed against the latest deepeval release.
+          labels: dependencies, automated

testrelic_deepeval-0.1.0/.github/workflows/publish-prod.yml

@@ -0,0 +1,31 @@
+name: Publish prod (PyPI)
+
+on:
+  push:
+    branches: [prod]
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    environment:
+      name: pypi
+      url: https://pypi.org/project/testrelic-deepeval/
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Build sdist + wheel
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

testrelic_deepeval-0.1.0/.github/workflows/publish-stage.yml

@@ -0,0 +1,34 @@
+name: Publish stage (TestPyPI)
+
+on:
+  push:
+    branches: [stage]
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/project/testrelic-deepeval/
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Build sdist + wheel
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build
+
+      - name: Publish to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          skip-existing: true

testrelic_deepeval-0.1.0/.gitignore

@@ -0,0 +1,74 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual environments
+.venv/
+venv/
+env/
+ENV/
+
+# Test/coverage caches
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+.pytest_cache/
+htmlcov/
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+
+# Type/lint caches
+.mypy_cache/
+.ruff_cache/
+.dmypy.json
+dmypy.json
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# TestRelic local state
+.testrelic/
+*.queue.json
+
+# Jupyter
+.ipynb_checkpoints
+
+# Local env
+.env
+.env.local

testrelic_deepeval-0.1.0/.python-version

@@ -0,0 +1 @@
+3.9

testrelic_deepeval-0.1.0/CLAUDE.md

@@ -0,0 +1,110 @@
+# testrelic-python-sdk - Development Guidelines
+
+## What this repo is
+
+The Python SDK that bridges DeepEval workflows into the TestRelic platform.
+Users `pip install testrelic-deepeval` and their `deepeval test run tests/`
+uploads to TestRelic's `/api/v1/evals/*` endpoints instead of Confident AI.
+
+Package name on PyPI: `testrelic-deepeval`. Import name: `testrelic`.
+
+## Stack
+
+- Python 3.9+ (target 3.9 in ruff/mypy for max compatibility)
+- httpx (sync client), pydantic v2, typer, platformdirs
+- pytest + respx for tests
+- ruff + mypy strict for lint/type
+- hatchling for build
+
+## Project structure
+
+```
+src/testrelic/
+  __init__.py            public re-exports
+  config.py              Config dataclass + precedence resolution
+  auth.py                ~/.testrelic/credentials.toml read/write
+  client.py              HttpClient (retries + offline queue) + drain_queue()
+  context.py             git + CI auto-detect
+  exceptions.py          TestRelicError / AuthError / ApiError
+  logger.py              "testrelic" logger
+  models/                Pydantic request/response models (camelCase fields)
+  deepeval/
+    plugin.py            pytest11 entry point
+    capture.py           reads DeepEval global TestRun, posts to /runs + /cases + /finalize
+    evaluate.py          drop-in wrapper for deepeval.evaluate()
+    translators/         TestRun -> TestRelic payload translators (defensive)
+  datasets/              pull/push/list
+  prompts/               Phase 3 stub
+  traces/                Phase 4 stub + OTel exporter shell
+  cli/                   typer app: login, logout, view, test, drain, version
+tests/
+  fixtures/              JSON golden files mirroring DeepEval TestRun.model_dump()
+  test_translators.py    Golden-file tests
+  test_capture.py        capture_and_upload orchestration
+  test_client.py         HTTP retries / queueing (respx)
+  test_plugin.py         entry-point smoke test
+  test_config.py         precedence resolution
+```
+
+## Commands
+
+| Command | Purpose |
+|---|---|
+| `pip install -e ".[dev]"` | Install with dev deps |
+| `ruff check src tests` | Lint |
+| `mypy src/testrelic` | Strict type check |
+| `pytest` | Run tests |
+| `pytest --cov=testrelic` | With coverage |
+| `python -m build` | Build wheel + sdist |
+
+## Release
+
+- `stage` branch -> TestPyPI via GH Actions OIDC (`.github/workflows/publish-stage.yml`)
+- `prod` branch -> PyPI via GH Actions OIDC (`.github/workflows/publish-prod.yml`)
+- Weekly `deepeval-compat.yml` bumps the deepeval pin and opens a PR if translator
+  tests still pass
+
+Bump `src/testrelic/_version.py` and `pyproject.toml` together; both must stay in sync.
+
+## Key design decisions
+
+- **TestRelic-native API.** This SDK calls TestRelic's `/api/v1/evals/*` endpoints. It
+  does NOT speak Confident AI's schema. The TestRelic API is camelCase JSON.
+- **Pytest plugin captures DeepEval in-memory.** No HTTP redirect, no monkey-patch -
+  we read `global_test_run_manager.get_test_run()` at
+  `pytest_sessionfinish(trylast=True)`. This means user code stays unchanged and
+  DeepEval's own behavior (printing results, returning eval objects) is preserved.
+- **Defensive translators.** DeepEval's Pydantic schema evolves between releases. Use
+  `getattr(obj, "field", default)` everywhere and keep golden-file fixtures under
+  `tests/fixtures/` so the weekly compat job catches breaking changes early.
+- **Offline queue.** Network failures write to `~/.testrelic/queue/`; calling
+  `testrelic drain` retries.
+- **Disabled-by-default when no key.** If `TESTRELIC_API_KEY` and the credentials file
+  are both absent, the plugin logs a single info line and otherwise no-ops. Test
+  runs never fail because of TestRelic.
+
+## Conventions
+
+- `from __future__ import annotations` at the top of every module
+- Imports sorted by ruff (`I` rule)
+- Public surface re-exported via `src/testrelic/__init__.py`
+- Pydantic models use camelCase field names directly (matching API JSON) with
+  `populate_by_name=True` and `extra="allow"` so the SDK degrades gracefully when
+  the server adds new optional fields
+- Don't comment inside functions unless explaining non-obvious behavior
+
+## Adding a new TestRelic endpoint
+
+1. Add the request/response model under `src/testrelic/models/`
+2. Add a thin function under the relevant module (`datasets/`, `traces/`, ...) that
+   constructs the model, calls `HttpClient.post|put|get`, and returns either the raw
+   dict or a parsed model
+3. Cover happy path + error envelope handling with a `respx`-mocked test
+4. Re-export the public surface from `__init__.py`
+
+## Adding a new DeepEval field
+
+1. Update the JSON fixture under `tests/fixtures/` to include the field
+2. Update the matching translator (`translators/test_case.py` or `metric_data.py`)
+3. Add an assertion to `tests/test_translators.py`
+4. Bump CHANGELOG, no need to bump SDK version unless the API contract changes

testrelic_deepeval-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 TestRelic
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

testrelic_deepeval-0.1.0/PKG-INFO

@@ -0,0 +1,184 @@
+Metadata-Version: 2.4
+Name: testrelic-deepeval
+Version: 0.1.0
+Summary: TestRelic SDK for DeepEval - LLM evaluation results land in TestRelic instead of Confident AI.
+Project-URL: Homepage, https://testrelic.ai
+Project-URL: Documentation, https://docs.testrelic.ai/deepeval
+Project-URL: Repository, https://github.com/testrelic-ai/testrelic-python-sdk
+Author-email: TestRelic <engineering@testrelic.ai>
+License: MIT
+License-File: LICENSE
+Keywords: deepeval,evaluation,llm,testing,testrelic
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Pytest
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.9
+Requires-Dist: httpx<1.0,>=0.27
+Requires-Dist: platformdirs>=4.0
+Requires-Dist: pydantic<3.0,>=2.6
+Requires-Dist: tomli-w>=1.0
+Requires-Dist: tomli>=2.0; python_version < '3.11'
+Requires-Dist: typer<1.0,>=0.12
+Provides-Extra: deepeval
+Requires-Dist: deepeval<4.0,>=2.0; extra == 'deepeval'
+Provides-Extra: dev
+Requires-Dist: deepeval<4.0,>=2.0; extra == 'dev'
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest-cov>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: respx>=0.21; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: otel
+Requires-Dist: opentelemetry-api>=1.25; extra == 'otel'
+Requires-Dist: opentelemetry-sdk>=1.25; extra == 'otel'
+Description-Content-Type: text/markdown
+
+# testrelic-deepeval
+
+The TestRelic Python SDK for [DeepEval](https://github.com/confident-ai/deepeval). Capture
+LLM evaluation results from your existing DeepEval test suite and ship them to your
+TestRelic workspace in one line of install.
+
+```bash
+pip install testrelic-deepeval
+testrelic login
+deepeval test run tests/
+```
+
+That's it. The pytest plugin captures DeepEval's in-memory `TestRun` at session finish
+and uploads cases + metrics to TestRelic's `/api/v1/evals/*` endpoints.
+
+## What this is (and isn't)
+
+- **Is**: a pytest plugin + Python client for sending DeepEval results to TestRelic
+- **Is**: a drop-in `testrelic.evaluate()` wrapper for programmatic eval runs
+- **Is**: a CLI (`testrelic login`, `testrelic test`, `testrelic view`)
+- **Is not**: a replacement for DeepEval (use DeepEval to author tests; we just receive
+  the results)
+- **Is not**: a Confident AI proxy — we use TestRelic's own API, not Confident's
+
+## Install
+
+```bash
+pip install testrelic-deepeval                  # core
+pip install "testrelic-deepeval[deepeval]"      # also pull deepeval if not already installed
+pip install "testrelic-deepeval[otel]"          # OTel tracing (Phase 4, preview)
+```
+
+Supported Python versions: 3.9, 3.10, 3.11, 3.12.
+
+## Authenticate
+
+```bash
+testrelic login --api-key tr_yourkey_here
+```
+
+Credentials are stored at `~/.testrelic/credentials.toml` with `0600` permissions on
+POSIX. Or pass credentials via environment:
+
+```bash
+export TESTRELIC_API_KEY=tr_yourkey
+export TESTRELIC_BASE_URL=https://platform.testrelic.ai/api/v1/evals  # only override if self-hosted
+```
+
+Precedence (highest to lowest): explicit `configure()` args → env vars → credentials
+file → built-in defaults.
+
+## Use with pytest
+
+No code changes needed. After install + login:
+
+```bash
+deepeval test run tests/
+# or, if you want the wrapper that double-checks credentials:
+testrelic test tests/
+```
+
+Every test run uploads as a new eval run in TestRelic, annotated with the current
+branch, commit, and CI run URL (auto-detected for GitHub Actions, GitLab, Jenkins,
+CircleCI, and Buildkite).
+
+## Use programmatically
+
+```python
+from deepeval.test_case import LLMTestCase
+from deepeval.metrics import AnswerRelevancyMetric
+from testrelic import evaluate
+
+results = evaluate(
+    test_cases=[LLMTestCase(input="Hi", actual_output="Hello")],
+    metrics=[AnswerRelevancyMetric(threshold=0.7)],
+)
+# results is whatever deepeval.evaluate() returns; upload is automatic
+```
+
+## Datasets
+
+```python
+from testrelic import datasets
+
+# Pull a dataset version into a deepeval EvaluationDataset
+ds = datasets.pull("customer-support-qa", label="latest")
+
+# Push goldens up to a new version
+datasets.push(
+    alias="customer-support-qa",
+    goldens=[{"input": "...", "expected_output": "..."}],
+    label="v2",
+    description="Refreshed Q1 2026 examples",
+)
+```
+
+## Migrating from Confident AI
+
+See [docs/migration-from-confident-ai.md](docs/migration-from-confident-ai.md). The
+TL;DR is:
+
+1. `pip install testrelic-deepeval`
+2. `testrelic login`
+3. Remove `CONFIDENT_API_KEY` from CI
+4. Run your existing tests unchanged
+
+Or, in one shot: `testrelic migrate-from-confident`.
+
+## Offline / flaky networks
+
+Failed uploads land in `~/.testrelic/queue/`. Replay them with:
+
+```bash
+testrelic drain
+```
+
+The plugin never fails your test run because of an upload error — uploads run at
+`pytest_sessionfinish(trylast=True)` and swallow exceptions with a warning log.
+
+## CLI
+
+| Command | Purpose |
+|---|---|
+| `testrelic login` | Save credentials |
+| `testrelic logout` | Remove credentials |
+| `testrelic test <path>` | Wrap `deepeval test run` with credential check |
+| `testrelic view [run_id]` | Open latest (or specific) eval run in browser |
+| `testrelic drain` | Replay queued offline uploads |
+| `testrelic version` | Print SDK version |
+| `testrelic migrate-from-confident` | Print migration steps |
+
+## Development
+
+```bash
+git clone https://github.com/testrelic-ai/testrelic-python-sdk
+cd testrelic-python-sdk
+pip install -e ".[dev]"
+ruff check src tests
+mypy src/testrelic
+pytest
+```
+
+## License
+
+MIT. See [LICENSE](LICENSE).

testrelic_deepeval-0.1.0/README.md

@@ -0,0 +1,145 @@
+# testrelic-deepeval
+
+The TestRelic Python SDK for [DeepEval](https://github.com/confident-ai/deepeval). Capture
+LLM evaluation results from your existing DeepEval test suite and ship them to your
+TestRelic workspace in one line of install.
+
+```bash
+pip install testrelic-deepeval
+testrelic login
+deepeval test run tests/
+```
+
+That's it. The pytest plugin captures DeepEval's in-memory `TestRun` at session finish
+and uploads cases + metrics to TestRelic's `/api/v1/evals/*` endpoints.
+
+## What this is (and isn't)
+
+- **Is**: a pytest plugin + Python client for sending DeepEval results to TestRelic
+- **Is**: a drop-in `testrelic.evaluate()` wrapper for programmatic eval runs
+- **Is**: a CLI (`testrelic login`, `testrelic test`, `testrelic view`)
+- **Is not**: a replacement for DeepEval (use DeepEval to author tests; we just receive
+  the results)
+- **Is not**: a Confident AI proxy — we use TestRelic's own API, not Confident's
+
+## Install
+
+```bash
+pip install testrelic-deepeval                  # core
+pip install "testrelic-deepeval[deepeval]"      # also pull deepeval if not already installed
+pip install "testrelic-deepeval[otel]"          # OTel tracing (Phase 4, preview)
+```
+
+Supported Python versions: 3.9, 3.10, 3.11, 3.12.
+
+## Authenticate
+
+```bash
+testrelic login --api-key tr_yourkey_here
+```
+
+Credentials are stored at `~/.testrelic/credentials.toml` with `0600` permissions on
+POSIX. Or pass credentials via environment:
+
+```bash
+export TESTRELIC_API_KEY=tr_yourkey
+export TESTRELIC_BASE_URL=https://platform.testrelic.ai/api/v1/evals  # only override if self-hosted
+```
+
+Precedence (highest to lowest): explicit `configure()` args → env vars → credentials
+file → built-in defaults.
+
+## Use with pytest
+
+No code changes needed. After install + login:
+
+```bash
+deepeval test run tests/
+# or, if you want the wrapper that double-checks credentials:
+testrelic test tests/
+```
+
+Every test run uploads as a new eval run in TestRelic, annotated with the current
+branch, commit, and CI run URL (auto-detected for GitHub Actions, GitLab, Jenkins,
+CircleCI, and Buildkite).
+
+## Use programmatically
+
+```python
+from deepeval.test_case import LLMTestCase
+from deepeval.metrics import AnswerRelevancyMetric
+from testrelic import evaluate
+
+results = evaluate(
+    test_cases=[LLMTestCase(input="Hi", actual_output="Hello")],
+    metrics=[AnswerRelevancyMetric(threshold=0.7)],
+)
+# results is whatever deepeval.evaluate() returns; upload is automatic
+```
+
+## Datasets
+
+```python
+from testrelic import datasets
+
+# Pull a dataset version into a deepeval EvaluationDataset
+ds = datasets.pull("customer-support-qa", label="latest")
+
+# Push goldens up to a new version
+datasets.push(
+    alias="customer-support-qa",
+    goldens=[{"input": "...", "expected_output": "..."}],
+    label="v2",
+    description="Refreshed Q1 2026 examples",
+)
+```
+
+## Migrating from Confident AI
+
+See [docs/migration-from-confident-ai.md](docs/migration-from-confident-ai.md). The
+TL;DR is:
+
+1. `pip install testrelic-deepeval`
+2. `testrelic login`
+3. Remove `CONFIDENT_API_KEY` from CI
+4. Run your existing tests unchanged
+
+Or, in one shot: `testrelic migrate-from-confident`.
+
+## Offline / flaky networks
+
+Failed uploads land in `~/.testrelic/queue/`. Replay them with:
+
+```bash
+testrelic drain
+```
+
+The plugin never fails your test run because of an upload error — uploads run at
+`pytest_sessionfinish(trylast=True)` and swallow exceptions with a warning log.
+
+## CLI
+
+| Command | Purpose |
+|---|---|
+| `testrelic login` | Save credentials |
+| `testrelic logout` | Remove credentials |
+| `testrelic test <path>` | Wrap `deepeval test run` with credential check |
+| `testrelic view [run_id]` | Open latest (or specific) eval run in browser |
+| `testrelic drain` | Replay queued offline uploads |
+| `testrelic version` | Print SDK version |
+| `testrelic migrate-from-confident` | Print migration steps |
+
+## Development
+
+```bash
+git clone https://github.com/testrelic-ai/testrelic-python-sdk
+cd testrelic-python-sdk
+pip install -e ".[dev]"
+ruff check src tests
+mypy src/testrelic
+pytest
+```
+
+## License
+
+MIT. See [LICENSE](LICENSE).