evalwise 0.1.0__tar.gz

evalwise-0.1.0/.gitignore

@@ -0,0 +1,38 @@
+# Byte-compiled
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution
+dist/
+build/
+*.egg-info/
+*.egg
+
+# Virtual environments
+venv/
+.venv/
+env/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.mypy_cache/
+
+# Model weights / downloaded artifacts
+*.pt
+*.pth
+*.ckpt
+*.safetensors
+*.bin
+
+# OS
+.DS_Store
+Thumbs.db

evalwise-0.1.0/PKG-INFO

@@ -0,0 +1,255 @@
+Metadata-Version: 2.5
+Name: evalwise
+Version: 0.1.0
+Summary: Deterministic-first AI evaluation for text, image, audio, and video
+Project-URL: Homepage, https://github.com/shreyaspj20/evalwise
+Project-URL: Documentation, https://github.com/shreyaspj20/evalwise#readme
+Project-URL: Repository, https://github.com/shreyaspj20/evalwise
+Author-email: Shreyas Reddy <shreyaspj20@gmail.com>
+License-Expression: MIT
+Keywords: ai,evaluation,llm,machine-learning,ml,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: click>=8.0
+Requires-Dist: httpx>=0.25
+Requires-Dist: jsonschema>=4.0
+Requires-Dist: langdetect>=1.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: rich>=13.0
+Requires-Dist: textstat>=0.7
+Provides-Extra: all
+Requires-Dist: decord>=0.6; extra == 'all'
+Requires-Dist: librosa>=0.10; extra == 'all'
+Requires-Dist: open-clip-torch>=2.20; extra == 'all'
+Requires-Dist: openai-whisper>=20231117; extra == 'all'
+Requires-Dist: opencv-python>=4.8; extra == 'all'
+Requires-Dist: pillow>=10.0; extra == 'all'
+Requires-Dist: sentence-transformers>=2.0; extra == 'all'
+Requires-Dist: soundfile>=0.12; extra == 'all'
+Requires-Dist: torch>=2.0; extra == 'all'
+Requires-Dist: transformers>=4.35; extra == 'all'
+Requires-Dist: ultralytics>=8.0; extra == 'all'
+Provides-Extra: audio
+Requires-Dist: librosa>=0.10; extra == 'audio'
+Requires-Dist: openai-whisper>=20231117; extra == 'audio'
+Requires-Dist: soundfile>=0.12; extra == 'audio'
+Provides-Extra: dev
+Requires-Dist: mypy>=1.5; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.21; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff>=0.1; extra == 'dev'
+Provides-Extra: embeddings
+Requires-Dist: sentence-transformers>=2.0; extra == 'embeddings'
+Provides-Extra: image
+Requires-Dist: open-clip-torch>=2.20; extra == 'image'
+Requires-Dist: pillow>=10.0; extra == 'image'
+Requires-Dist: torch>=2.0; extra == 'image'
+Requires-Dist: transformers>=4.35; extra == 'image'
+Requires-Dist: ultralytics>=8.0; extra == 'image'
+Provides-Extra: nli
+Requires-Dist: torch>=2.0; extra == 'nli'
+Requires-Dist: transformers>=4.35; extra == 'nli'
+Provides-Extra: video
+Requires-Dist: decord>=0.6; extra == 'video'
+Requires-Dist: opencv-python>=4.8; extra == 'video'
+Description-Content-Type: text/markdown
+
+# EvalWise
+
+**Deterministic-first AI evaluation for text, image, audio, and video.**
+
+The eval SDK that doesn't default to "ask another AI if this is good."
+
+```bash
+pip install evalwise
+```
+
+## Why EvalWise?
+
+Most eval tools jump straight to LLM-as-judge. That's:
+- **Expensive** — every eval is another API call
+- **Non-deterministic** — same input, different scores
+- **Ungrounded** — "Score: 4/5" tells you nothing
+
+EvalWise flips the default: **deterministic checks first, LLM-judge only when you have to.**
+
+## Quick Start
+
+```python
+from evalwise import Suite, Assert
+
+suite = Suite("summarization")
+
+@suite.test
+def test_format(response: str):
+    Assert.bullet_count(response, exactly=3)
+    Assert.word_count(response, max=200)
+    Assert.json_valid(response)
+
+@suite.test
+def test_factuality(response: str, source: str):
+    Assert.entails(response, source=source)
+    Assert.no_contradiction(response, source=source)
+
+# Run against a dataset
+results = suite.run(dataset="./golden_set.json")
+results.assert_pass_rate(threshold=0.95)
+```
+
+## CLI
+
+```bash
+# Run eval suite
+evalwise run tests/test_summary.py --dataset golden.json
+
+# Run in CI (exit code 1 on failure)
+evalwise run tests/ --ci --threshold 0.95
+
+# Create sample eval file
+evalwise init
+```
+
+## Text Assertions
+
+| Assertion | What it checks |
+|-----------|----------------|
+| `contains(text, substring)` | Substring present |
+| `not_contains(text, substring)` | Substring absent |
+| `regex(text, pattern)` | Pattern match |
+| `json_valid(text)` | Parseable JSON |
+| `json_schema(text, schema)` | Matches JSON schema |
+| `word_count(text, min, max)` | Word count in range |
+| `bullet_count(text, exactly)` | Bullet point count |
+| `readability(text, min_score)` | Flesch Reading Ease |
+| `language_is(text, "en")` | Correct language |
+| `code_parses(text, "python")` | Valid syntax |
+| `code_runs(text)` | Executes without error |
+| `entails(text, source)` | Follows from source (NLI) |
+| `no_contradiction(text, source)` | No contradictions |
+| `embedding_similarity(text, ref)` | Semantic similarity |
+| `urls_valid(text)` | All URLs return 2xx |
+
+## Image Assertions
+
+Image evals cover prompt alignment, object detection, resolution/aspect ratio, NSFW safety, and similarity. Object detection uses **YOLO** via `ultralytics`.
+
+```python
+from evalwise.image import ImageAssert
+
+# Prompt alignment with CLIP
+ImageAssert.clip_score(image, "a cat on a couch", threshold=0.25)
+
+# Object detection
+ImageAssert.contains_object(image, "cat", confidence=0.5)
+ImageAssert.object_count(image, "person", exactly=2, confidence=0.5)
+
+# Metadata
+ImageAssert.resolution_is(image, width=1024, height=1024)
+ImageAssert.resolution_min(image, width=512, height=512)
+ImageAssert.aspect_ratio(image, ratio=1.0, tolerance=0.1)
+ImageAssert.format_is(image, "PNG")
+
+# Safety
+ImageAssert.nsfw_below(image, threshold=0.1)
+
+# Similarity to a reference image
+ImageAssert.image_similarity(image, reference, threshold=0.8)
+```
+
+### Image generation example
+
+`examples/test_image_generation.py` shows a complete eval suite. The dataset can include per-image thresholds and object lists:
+
+```json
+{
+  "image_path": "./examples/sample_cat.jpeg",
+  "prompt": "a person sitting on a couch with a dog and a cat",
+  "min_width": 200,
+  "min_height": 100,
+  "required_objects": ["dog"],
+  "confidence": 0.25
+}
+```
+
+## Audio Assertions
+
+Audio assertions that use Whisper (`transcription_contains`, `transcription_equals`, `language_is`) require **ffmpeg** to be installed on your system in addition to the `evalwise[audio]` Python dependencies.
+
+```python
+from evalwise.audio import AudioAssert
+
+AudioAssert.transcription_contains(audio, "hello world")
+AudioAssert.transcription_equals(audio, expected_text)
+AudioAssert.language_is(audio, "en")
+AudioAssert.duration_between(audio, min_sec=5, max_sec=30)
+AudioAssert.sample_rate_is(audio, hz=44100)
+AudioAssert.no_silence(audio, max_silence_sec=1.0)
+```
+
+## Installation
+
+```bash
+# Core (text assertions)
+pip install evalwise
+
+# With image support
+pip install evalwise[image]
+
+# With audio support (also requires ffmpeg system binary)
+pip install evalwise[audio]
+
+# On macOS: brew install ffmpeg
+# On Ubuntu: sudo apt install ffmpeg
+# On Windows: winget install Gyan.FFmpeg
+
+# Everything
+pip install evalwise[all]
+```
+
+### What each extra gives you
+
+| Extra | Capabilities enabled | Example tests |
+|---|---|---|
+| *(none)* | 22 of 25 text assertions | `test_basic.py` |
+| `[nli]` | `Assert.entails`, `Assert.no_contradiction` | `test_summarization.py` |
+| `[embeddings]` | `Assert.embedding_similarity` | — |
+| `[image]` | CLIP, YOLO object detection, NSFW, resolution, similarity | `test_image_generation.py` |
+| `[audio]` | Whisper transcription, language, duration, sample rate | `test_audio.py` |
+| `[video]` | Video assertions (extra only, no example yet) | — |
+| `[all]` | All of the above | — |
+
+### System dependencies
+
+Some examples also require system binaries:
+
+| Capability | System binary | Install |
+|---|---|---|
+| Audio transcription | `ffmpeg` | `brew install ffmpeg` / `apt install ffmpeg` |
+
+## Philosophy
+
+1. **Deterministic by default** — Same input, same result. Always.
+2. **Cheap first** — Check format, length, syntax before calling models.
+3. **Grounded scores** — Know exactly why something failed.
+4. **LLM-judge as last resort** — Only for truly subjective criteria.
+
+## Comparison
+
+| | EvalWise | Promptfoo | Braintrust | LangSmith |
+|---|--------|-----------|------------|-----------|
+| Deterministic-first | ✓ | Partial | ✗ | ✗ |
+| Image/Audio evals | ✓ | ✗ | ✗ | ✗ |
+| Python-native | ✓ | YAML | SDK | SDK |
+| No account required | ✓ | ✓ | ✗ | ✗ |
+| OSS | ✓ | ✓ | Partial | ✗ |
+
+## License
+
+MIT

evalwise-0.1.0/README.md

@@ -0,0 +1,192 @@
+# EvalWise
+
+**Deterministic-first AI evaluation for text, image, audio, and video.**
+
+The eval SDK that doesn't default to "ask another AI if this is good."
+
+```bash
+pip install evalwise
+```
+
+## Why EvalWise?
+
+Most eval tools jump straight to LLM-as-judge. That's:
+- **Expensive** — every eval is another API call
+- **Non-deterministic** — same input, different scores
+- **Ungrounded** — "Score: 4/5" tells you nothing
+
+EvalWise flips the default: **deterministic checks first, LLM-judge only when you have to.**
+
+## Quick Start
+
+```python
+from evalwise import Suite, Assert
+
+suite = Suite("summarization")
+
+@suite.test
+def test_format(response: str):
+    Assert.bullet_count(response, exactly=3)
+    Assert.word_count(response, max=200)
+    Assert.json_valid(response)
+
+@suite.test
+def test_factuality(response: str, source: str):
+    Assert.entails(response, source=source)
+    Assert.no_contradiction(response, source=source)
+
+# Run against a dataset
+results = suite.run(dataset="./golden_set.json")
+results.assert_pass_rate(threshold=0.95)
+```
+
+## CLI
+
+```bash
+# Run eval suite
+evalwise run tests/test_summary.py --dataset golden.json
+
+# Run in CI (exit code 1 on failure)
+evalwise run tests/ --ci --threshold 0.95
+
+# Create sample eval file
+evalwise init
+```
+
+## Text Assertions
+
+| Assertion | What it checks |
+|-----------|----------------|
+| `contains(text, substring)` | Substring present |
+| `not_contains(text, substring)` | Substring absent |
+| `regex(text, pattern)` | Pattern match |
+| `json_valid(text)` | Parseable JSON |
+| `json_schema(text, schema)` | Matches JSON schema |
+| `word_count(text, min, max)` | Word count in range |
+| `bullet_count(text, exactly)` | Bullet point count |
+| `readability(text, min_score)` | Flesch Reading Ease |
+| `language_is(text, "en")` | Correct language |
+| `code_parses(text, "python")` | Valid syntax |
+| `code_runs(text)` | Executes without error |
+| `entails(text, source)` | Follows from source (NLI) |
+| `no_contradiction(text, source)` | No contradictions |
+| `embedding_similarity(text, ref)` | Semantic similarity |
+| `urls_valid(text)` | All URLs return 2xx |
+
+## Image Assertions
+
+Image evals cover prompt alignment, object detection, resolution/aspect ratio, NSFW safety, and similarity. Object detection uses **YOLO** via `ultralytics`.
+
+```python
+from evalwise.image import ImageAssert
+
+# Prompt alignment with CLIP
+ImageAssert.clip_score(image, "a cat on a couch", threshold=0.25)
+
+# Object detection
+ImageAssert.contains_object(image, "cat", confidence=0.5)
+ImageAssert.object_count(image, "person", exactly=2, confidence=0.5)
+
+# Metadata
+ImageAssert.resolution_is(image, width=1024, height=1024)
+ImageAssert.resolution_min(image, width=512, height=512)
+ImageAssert.aspect_ratio(image, ratio=1.0, tolerance=0.1)
+ImageAssert.format_is(image, "PNG")
+
+# Safety
+ImageAssert.nsfw_below(image, threshold=0.1)
+
+# Similarity to a reference image
+ImageAssert.image_similarity(image, reference, threshold=0.8)
+```
+
+### Image generation example
+
+`examples/test_image_generation.py` shows a complete eval suite. The dataset can include per-image thresholds and object lists:
+
+```json
+{
+  "image_path": "./examples/sample_cat.jpeg",
+  "prompt": "a person sitting on a couch with a dog and a cat",
+  "min_width": 200,
+  "min_height": 100,
+  "required_objects": ["dog"],
+  "confidence": 0.25
+}
+```
+
+## Audio Assertions
+
+Audio assertions that use Whisper (`transcription_contains`, `transcription_equals`, `language_is`) require **ffmpeg** to be installed on your system in addition to the `evalwise[audio]` Python dependencies.
+
+```python
+from evalwise.audio import AudioAssert
+
+AudioAssert.transcription_contains(audio, "hello world")
+AudioAssert.transcription_equals(audio, expected_text)
+AudioAssert.language_is(audio, "en")
+AudioAssert.duration_between(audio, min_sec=5, max_sec=30)
+AudioAssert.sample_rate_is(audio, hz=44100)
+AudioAssert.no_silence(audio, max_silence_sec=1.0)
+```
+
+## Installation
+
+```bash
+# Core (text assertions)
+pip install evalwise
+
+# With image support
+pip install evalwise[image]
+
+# With audio support (also requires ffmpeg system binary)
+pip install evalwise[audio]
+
+# On macOS: brew install ffmpeg
+# On Ubuntu: sudo apt install ffmpeg
+# On Windows: winget install Gyan.FFmpeg
+
+# Everything
+pip install evalwise[all]
+```
+
+### What each extra gives you
+
+| Extra | Capabilities enabled | Example tests |
+|---|---|---|
+| *(none)* | 22 of 25 text assertions | `test_basic.py` |
+| `[nli]` | `Assert.entails`, `Assert.no_contradiction` | `test_summarization.py` |
+| `[embeddings]` | `Assert.embedding_similarity` | — |
+| `[image]` | CLIP, YOLO object detection, NSFW, resolution, similarity | `test_image_generation.py` |
+| `[audio]` | Whisper transcription, language, duration, sample rate | `test_audio.py` |
+| `[video]` | Video assertions (extra only, no example yet) | — |
+| `[all]` | All of the above | — |
+
+### System dependencies
+
+Some examples also require system binaries:
+
+| Capability | System binary | Install |
+|---|---|---|
+| Audio transcription | `ffmpeg` | `brew install ffmpeg` / `apt install ffmpeg` |
+
+## Philosophy
+
+1. **Deterministic by default** — Same input, same result. Always.
+2. **Cheap first** — Check format, length, syntax before calling models.
+3. **Grounded scores** — Know exactly why something failed.
+4. **LLM-judge as last resort** — Only for truly subjective criteria.
+
+## Comparison
+
+| | EvalWise | Promptfoo | Braintrust | LangSmith |
+|---|--------|-----------|------------|-----------|
+| Deterministic-first | ✓ | Partial | ✗ | ✗ |
+| Image/Audio evals | ✓ | ✗ | ✗ | ✗ |
+| Python-native | ✓ | YAML | SDK | SDK |
+| No account required | ✓ | ✓ | ✗ | ✗ |
+| OSS | ✓ | ✓ | Partial | ✗ |
+
+## License
+
+MIT

evalwise-0.1.0/examples/audio_data.json

@@ -0,0 +1,6 @@
+[
+  {
+    "audio_path": "./examples/sample_audio.wav",
+    "expected_text": ""
+  }
+]

evalwise-0.1.0/examples/code_data.json

@@ -0,0 +1,14 @@
+[
+  {
+    "response": "```python\nprint('Hello, World!')\n```",
+    "expected_output": "Hello, World!"
+  },
+  {
+    "response": "```python\nx = 5\ny = 3\nprint(x + y)\n```",
+    "expected_output": "8"
+  },
+  {
+    "response": "```python\ndef greet(name):\n    return f'Hello, {name}!'\nprint(greet('EvalWise'))\n```",
+    "expected_output": "Hello, EvalWise!"
+  }
+]

evalwise-0.1.0/examples/image_data.json

@@ -0,0 +1,12 @@
+[
+  {
+    "image_path": "./examples/sample_cat.jpeg",
+    "prompt": "a person in an orange shirt sitting on a couch holding a brown and white dog and petting a grey and white tabby cat",
+    "min_width": 200,
+    "min_height": 100,
+    "ratio": 1.4,
+    "tolerance": 0.1,
+    "required_objects": ["dog"],
+    "confidence": 0.25
+  }
+]

evalwise-0.1.0/examples/summarization_data.json

@@ -0,0 +1,14 @@
+[
+  {
+    "response": "- The study found that exercise improves mental health\n- Participants who exercised 3x/week showed 40% reduction in anxiety\n- Results were consistent across age groups",
+    "source": "A new study published in the Journal of Mental Health examined 500 participants over 6 months. Researchers found that regular exercise significantly improves mental health outcomes. Participants who exercised three times per week showed a 40% reduction in anxiety symptoms compared to the control group. The positive effects were observed consistently across all age groups from 18 to 65."
+  },
+  {
+    "response": "- Climate change is accelerating faster than predicted\n- Arctic ice loss reached record levels in 2024\n- Scientists call for immediate action",
+    "source": "New data from NASA satellites reveals that climate change is accelerating at a rate faster than previous models predicted. Arctic sea ice reached its lowest extent on record in 2024, losing 15% more ice than the previous year. Climate scientists are calling for immediate and aggressive action to reduce carbon emissions, warning that current policies are insufficient to prevent catastrophic warming."
+  },
+  {
+    "response": "- Apple announced the iPhone 20 with holographic display\n- Battery life extended to 5 days\n- Price starts at $1,299",
+    "source": "Apple unveiled its latest smartphone, the iPhone 16, at its annual fall event. The new device features an improved OLED display with higher brightness and a new A18 chip. Battery life has been improved by 20% over the previous generation. The base model starts at $999, with the Pro version at $1,199."
+  }
+]

evalwise-0.1.0/examples/test_audio.py

@@ -0,0 +1,29 @@
+"""Example: Evaluating speech/audio generation."""
+
+from evalwise import Suite
+from evalwise.audio import AudioAssert
+
+suite = Suite("audio_evals")
+
+
+@suite.test
+def test_transcription(audio_path: str, expected_text: str):
+    """Check if speech contains expected text (using Whisper)."""
+    AudioAssert.transcription_contains(audio_path, expected_text)
+
+
+@suite.test
+def test_language(audio_path: str):
+    """Check audio is in English."""
+    AudioAssert.language_is(audio_path, "en")
+
+
+@suite.test
+def test_duration(audio_path: str):
+    """Check audio duration is reasonable."""
+    AudioAssert.duration_between(audio_path, min_sec=1, max_sec=60)
+
+
+# To run (install deps first):
+#   pip install openai-whisper librosa soundfile
+#   evalwise run examples/test_audio.py --dataset examples/audio_data.json

evalwise-0.1.0/examples/test_basic.py

@@ -0,0 +1,26 @@
+"""Basic example - no external dependencies required."""
+
+from evalwise import Suite, Assert
+
+suite = Suite("basic_evals")
+
+
+@suite.test
+def test_format(response: str):
+    """Test basic format checks."""
+    # These use only built-in Python - no dependencies
+    Assert.contains(response, "-")  # Has bullet points
+    Assert.word_count(response, min=5, max=100)
+    Assert.line_count(response, min=1, max=10)
+
+
+@suite.test
+def test_structure(response: str):
+    """Test structure checks."""
+    Assert.bullet_count(response, min=1)
+    Assert.not_contains(response, "ERROR")
+    Assert.not_contains(response, "undefined")
+
+
+# To run:
+# evalwise run examples/test_basic.py --dataset examples/summarization_data.json

evalwise-0.1.0/examples/test_code_generation.py

@@ -0,0 +1,53 @@
+"""Example: Evaluating code generation."""
+
+import io
+import re
+import sys
+
+from evalwise import Assert, Suite
+
+suite = Suite("code_generation_evals")
+
+
+def _extract_code(response: str) -> str:
+    """Extract code from a markdown block if present."""
+    code_match = re.search(r"```(?:\w+)?\n(.*?)```", response, re.DOTALL)
+    return code_match.group(1) if code_match else response
+
+
+@suite.test
+def test_syntax(response: str):
+    """Ensure generated code is syntactically valid."""
+    Assert.code_parses(response, language="python")
+
+
+@suite.test
+def test_execution(response: str):
+    """Ensure generated code executes without error."""
+    code = _extract_code(response)
+    Assert.code_runs(code)
+
+
+@suite.test
+def test_output(response: str, expected_output: str):
+    """Ensure code produces expected output."""
+    code = _extract_code(response)
+
+    old_stdout = sys.stdout
+    sys.stdout = buffer = io.StringIO()
+    try:
+        exec(code)
+        actual_output = buffer.getvalue().strip()
+    finally:
+        sys.stdout = old_stdout
+
+    Assert.custom(
+        passed=actual_output == expected_output,
+        name="output_matches",
+        expected=expected_output,
+        actual=actual_output,
+    )
+
+
+# To run:
+# evalwise run examples/test_code_generation.py --dataset examples/code_data.json