promptum 0.0.1__py3-none-any.whl

promptum-0.0.1.dist-info/METADATA

@@ -0,0 +1,280 @@
+Metadata-Version: 2.4
+Name: promptum
+Version: 0.0.1
+Summary: Async LLM benchmarking library with protocol-based extensibility
+Project-URL: Homepage, https://github.com/deyna256/promptum
+Project-URL: Repository, https://github.com/deyna256/promptum
+Project-URL: Issues, https://github.com/deyna256/promptum/issues
+Author-email: deyna256 <literallybugcreator@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2026 Ivan Deyna
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: anthropic,async,benchmarking,llm,openai
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.13
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: jinja2>=3.1.0
+Requires-Dist: pyyaml>=6.0
+Description-Content-Type: text/markdown
+
+# promptum
+
+<div align="center">
+
+![Python 3.13+](https://img.shields.io/badge/Python-3.13+-blue?style=for-the-badge&logo=python)
+![Async](https://img.shields.io/badge/Async-First-green?style=for-the-badge)
+![License: MIT](https://img.shields.io/badge/License-MIT-yellow?style=for-the-badge)
+
+**Benchmark LLMs Like a Pro. In 5 Lines of Code.**
+
+Stop writing boilerplate to test LLMs. Start getting results.
+
+</div>
+
+---
+
+## What's This?
+
+A dead-simple Python library for benchmarking LLM providers. Write tests once, run them across any model, get beautiful reports.
+
+```python
+benchmark = Benchmark(provider=client, name="my_test")
+benchmark.add_test(TestCase(
+    prompt="What is 2+2?",
+    model="gpt-3.5-turbo",
+    validator=Contains("4")
+))
+report = await benchmark.run_async()
+```
+
+That's it. No setup. No config files. Just results.
+
+---
+
+## Why You Need This
+
+**Before promptum:**
+```python
+# Custom API client for each provider
+openai_client = OpenAI(api_key=...)
+anthropic_client = Anthropic(api_key=...)
+
+# Manual validation logic
+if "correct answer" not in response:
+    failed_tests.append(...)
+
+# Track metrics yourself
+latency = end_time - start_time
+tokens = response.usage.total_tokens
+
+# Write your own retry logic
+for attempt in range(max_retries):
+    try:
+        response = client.chat.completions.create(...)
+        break
+    except Exception:
+        sleep(2 ** attempt)
+
+# Export results manually
+json.dump(results, open("results.json", "w"))
+```
+
+**After promptum:**
+```python
+report = await benchmark.run_async()
+HTMLSerializer().serialize(report)  # Beautiful HTML report
+```
+
+---
+
+## Quick Start
+
+```bash
+pip install promptum  # (or: uv pip install promptum)
+export OPENROUTER_API_KEY="your-key"
+```
+
+```python
+import asyncio
+from promptum import Benchmark, TestCase, OpenRouterClient, Contains
+
+async def main():
+    async with OpenRouterClient(api_key="your-key") as client:
+        benchmark = Benchmark(provider=client, name="quick_test")
+
+        benchmark.add_test(TestCase(
+            name="basic_math",
+            prompt="What is 15 * 7? Reply with just the number.",
+            model="openai/gpt-3.5-turbo",
+            validator=Contains("105")
+        ))
+
+        report = await benchmark.run_async()
+        summary = report.get_summary()
+
+        print(f"✓ {summary['passed']}/{summary['total']} tests passed")
+        print(f"⚡ {summary['avg_latency_ms']:.0f}ms average")
+        print(f"💰 ${summary['total_cost_usd']:.6f} total cost")
+
+asyncio.run(main())
+```
+
+Run it:
+```bash
+python your_script.py
+```
+
+---
+
+## What You Get
+
+✅ **One API for 100+ Models** - OpenRouter support out of the box (OpenAI, Anthropic, Google, etc.)
+✅ **Smart Validation** - ExactMatch, Contains, Regex, JsonSchema, or write your own
+✅ **Automatic Retries** - Exponential/linear backoff with configurable attempts
+✅ **Metrics Tracking** - Latency, tokens, cost - automatically captured
+✅ **Beautiful Reports** - JSON, YAML, or interactive HTML with charts
+✅ **Async by Default** - Run 100 tests in parallel without breaking a sweat
+✅ **Type Safe** - Full type hints, catches errors before runtime
+✅ **Zero Config** - No YAML files, no setup scripts, just Python
+
+---
+
+## Real Example
+
+Compare GPT-4 vs Claude on your tasks:
+
+```python
+from promptum import Benchmark, TestCase, ExactMatch, Contains, Regex
+
+tests = [
+    TestCase(
+        name="json_output",
+        prompt='Output JSON: {"status": "ok"}',
+        model="openai/gpt-4",
+        validator=Regex(r'\{"status":\s*"ok"\}')
+    ),
+    TestCase(
+        name="json_output",
+        prompt='Output JSON: {"status": "ok"}',
+        model="anthropic/claude-3-5-sonnet",
+        validator=Regex(r'\{"status":\s*"ok"\}')
+    ),
+    TestCase(
+        name="creative_writing",
+        prompt="Write a haiku about Python",
+        model="openai/gpt-4",
+        validator=Contains("Python", case_sensitive=False)
+    ),
+]
+
+benchmark.add_tests(tests)
+report = await benchmark.run_async()
+
+# Export as HTML
+from promptum import HTMLSerializer
+html = HTMLSerializer().serialize(report)
+open("comparison.html", "w").write(html)
+```
+
+Open `comparison.html` in your browser - see side-by-side model performance with charts.
+
+---
+
+## Use Cases
+
+**🔬 Model Evaluation** - Compare GPT-4, Claude, Gemini on your specific tasks
+**🎯 Prompt Engineering** - Test 100 prompt variations, find what works
+**⚡ Latency Testing** - Measure real-world response times across providers
+**💰 Cost Analysis** - Track spending per model/task before production
+**🔄 Regression Testing** - Ensure model updates don't break your prompts
+**📊 A/B Testing** - Data-driven model selection for your product
+
+---
+
+## Requirements
+
+- Python 3.13+
+- An OpenRouter API key (or implement your own provider)
+
+That's it. No Docker, no complex setup.
+
+---
+
+## Why Protocol-Based?
+
+Most libraries force inheritance:
+```python
+class MyProvider(BaseProvider):  # Tightly coupled
+    def generate(self): ...
+```
+
+We use protocols (structural typing):
+```python
+class MyProvider:  # No inheritance needed
+    async def generate(self) -> tuple[str, Metrics]:
+        # Your implementation
+        return response, metrics
+
+# It just works
+benchmark = Benchmark(provider=MyProvider())
+```
+
+Cleaner. More flexible. More Pythonic.
+
+---
+
+## Contributing
+
+Found a bug? Want a feature? PRs welcome!
+
+```bash
+# Development setup
+git clone https://github.com/yourusername/promptum.git
+cd promptum
+just sync       # Install dependencies
+just test       # Run tests
+
+# Development commands
+just lint       # Check code style
+just format     # Format code
+just typecheck  # Type checking
+```
+
+---
+
+## License
+
+MIT - do whatever you want with it.
+
+---
+
+<div align="center">
+
+**[⭐ Star on GitHub](https://github.com/yourusername/promptum)** | **[🐛 Report Bug](https://github.com/yourusername/promptum/issues)** | **[💡 Request Feature](https://github.com/yourusername/promptum/issues)**
+
+Made for developers who value their time.
+
+</div>

promptum-0.0.1.dist-info/RECORD

@@ -0,0 +1,32 @@
+promptum/__init__.py,sha256=AjeGgmIbpp9Uv-0ybq6knejEJMK-Dnn_-fV9Z86Bp74,932
+promptum/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+promptum/benchmark/__init__.py,sha256=NJYiXm6wVFKMloxKNAXMY4H3bMQORTtLh6__nYWYWa0,131
+promptum/benchmark/benchmark.py,sha256=3enQSACdLwHW78fqSZj0Un3r7_Ua3V-MjfbEIIKFSWs,1589
+promptum/benchmark/report.py,sha256=ol_UO8rw43zbQxhs2o4AwYN5TP7O_Apa77V-pZKq6Uw,2754
+promptum/core/__init__.py,sha256=mqajsOdUBNJfcR2krxpwa7rM_wd88vJaAov-9SnVm68,294
+promptum/core/metrics.py,sha256=FnS10nHFjQ5Clj5X21C_nW6zAUJU_ZHt0s2fLgp6L28,427
+promptum/core/result.py,sha256=nyuVMQFY6DmZwzpgqDPsj0FaAuairpKLJ-0be5WQtTg,472
+promptum/core/retry.py,sha256=mA_RRz9_9J_mge_AUd9f1A-gACOxZLGTI8vTIstAr8s,538
+promptum/core/test_case.py,sha256=YNlVNj7FkoCyBFb2N0Dzrhce6o3DzUtke4PR6WoXhZo,593
+promptum/execution/__init__.py,sha256=fUZa7Bo7yn921sl49cS6TCGsG-lOUNVdhdeRsIa5vCc,67
+promptum/execution/runner.py,sha256=sP3uDu2VDLxFi9BkltMHwsyMuCXnz4oP1kVN28KpVZ0,2434
+promptum/providers/__init__.py,sha256=OW-CK198wOV7_bz_keOaxxQeRlFPZgINQcVJUZq_uus,169
+promptum/providers/openrouter.py,sha256=owquGxHaTB-pZ8jr06l4HouETuFj1lEg92oGX2mM5uo,4601
+promptum/providers/protocol.py,sha256=vdTGAGKN3FzThHLwyMMWicU87_LpW-gn0cM3vMcWiEY,488
+promptum/serialization/__init__.py,sha256=0dlpgF3dngaw_oR4mg7nuc4Z_VFVl2bATmhe2mHA9T4,319
+promptum/serialization/base.py,sha256=JnB4zb7D4oy44k6ndbJu3Xw1PVLpY_9-Y7k3Et2p43g,1851
+promptum/serialization/html.py,sha256=kJEd2s6fVfFHH7snJWrD5RGaUW66x3vtMKGMJ_ekmcI,1901
+promptum/serialization/json.py,sha256=koqgr5_WHmrpWUOCq6rWXoC07um3mkDDaob2k9vkEK8,870
+promptum/serialization/protocol.py,sha256=MZeMYt_HZJIYSyrRd_ZYbEJXDiXLMuJ5tosAeHLxpTM,353
+promptum/serialization/report_template.html,sha256=RC8qSLzolqWkWBIGfyhPtPkRWM7_0JkauEWPkaKiB9A,10802
+promptum/serialization/yaml.py,sha256=50A612OkX2L3EjhxTZJMZQb5zL8-2PmwcBjjNUhCWsA,528
+promptum/storage/__init__.py,sha256=QWOP5Al43WmmQ_kFCM9JGi8amXJzO_pR-x5AKDNy4ds,153
+promptum/storage/file.py,sha256=gnNBpNBQ_NeAWn7P2itsw2L99AxS7zOd8Nef6PyYxlk,5750
+promptum/storage/protocol.py,sha256=_NpkJzOQB_98Ud_TA_ZYubHf3o2DDXGMveRN3kRyYKI,517
+promptum/validation/__init__.py,sha256=mhykyxaIwn2PJh2RXAi0fi2NRIveFmlC5bg1nyCbfVU,252
+promptum/validation/protocol.py,sha256=xqxm23YX6eNeZHKMLMZ-Wz8iQKn4ZRzAI5Xryxg0uq4,418
+promptum/validation/validators.py,sha256=3lJwSMhhWb9x8BK_-S0FJBj7PFgno79II_i3Z1mCKTs,3217
+promptum-0.0.1.dist-info/METADATA,sha256=vt_PN0Ns0JuJalM7p8hJZsz-Y2hwQrbHZ4Jacy7P6L8,8083
+promptum-0.0.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+promptum-0.0.1.dist-info/licenses/LICENSE,sha256=Fgn285H5Vy9diOlqO1TzS3hD97WcdF6-GFHvUcFNtmg,1067
+promptum-0.0.1.dist-info/RECORD,,

promptum-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any

promptum-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ivan Deyna
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.