trustwise 0.1.0a1__tar.gz

trustwise-0.1.0a1/.githooks/pre-push

@@ -0,0 +1,48 @@
+#!/bin/sh
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+NC='\033[0m' # No Color
+
+echo "Running pre-push checks..."
+
+# Function to check if a command exists
+command_exists() {
+    command -v "$1" >/dev/null 2>&1
+}
+
+# Check if uv is installed
+if ! command_exists uv; then
+    echo -e "${RED}Error: uv is not installed. Please install it first:${NC}"
+    echo "curl -LsSf https://astral.sh/uv/install.sh | sh"
+    exit 1
+fi
+
+# Install dependencies if needed
+echo "Installing dependencies..."
+uv pip install --system -e ".[dev]"
+
+# Run pytest with coverage
+echo "Running tests..."
+if ! pytest tests/ -v --cov=trustwise --cov-report=term-missing; then
+    echo -e "${RED}Tests failed!${NC}"
+    exit 1
+fi
+
+# Run ruff linting
+echo "Running linting checks..."
+if ! ruff check src tests; then
+    echo -e "${RED}Linting failed!${NC}"
+    exit 1
+fi
+
+# Build documentation
+echo "Building documentation..."
+if ! sphinx-build -b html docs/source docs/_build/html; then
+    echo -e "${RED}Documentation build failed!${NC}"
+    exit 1
+fi
+
+echo -e "${GREEN}All checks passed!${NC}"
+exit 0 

trustwise-0.1.0a1/.github/workflows/build-and-test.yml

@@ -0,0 +1,56 @@
+name: Build and Test Package
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    
+    - name: Install uv
+      run: |
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+    
+    - name: Build wheel
+      run: |
+        uv build --wheel
+    
+    - name: Create isolated environment and test installation
+      run: |
+        # Create a new virtual environment
+        python -m venv test_env
+        source test_env/bin/activate
+        
+        # Install the wheel
+        pip install dist/*.whl
+        
+        # Test importing the package and key components
+        python -c "
+        import trustwise
+        print('Successfully imported trustwise')
+        
+        # Test importing main SDK components
+        from trustwise.sdk import TrustwiseSDK
+        from trustwise.sdk.config import TrustwiseConfig
+        print('Successfully imported SDK components')
+        
+        # Verify package metadata
+        assert hasattr(trustwise, '__version__')
+        assert isinstance(trustwise.__version__, str)
+        print('Successfully verified package metadata')
+        "

trustwise-0.1.0a1/.github/workflows/ci.yml

@@ -0,0 +1,102 @@
+name: CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+# Add permissions at the workflow level
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    
+    - name: Install uv
+      run: |
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+    
+    - name: Install dependencies
+      run: |
+        uv pip install --system -e ".[dev]"
+    
+    - name: Run tests with tox
+      run: |
+        tox -e py${{ matrix.python-version == '3.11' && '311' || '312' }}
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+    
+    - name: Install uv
+      run: |
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+    
+    - name: Install dependencies
+      run: |
+        uv pip install --system -e ".[dev]"
+    
+    - name: Run linting with tox
+      run: |
+        tox -e lint
+
+  docs:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+    
+    - name: Install uv
+      run: |
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+    
+    - name: Install dependencies
+      run: |
+        uv pip install --system -e ".[dev]"
+    
+    - name: Build documentation with tox
+      run: |
+        tox -e docs
+
+    - name: Setup Pages
+      uses: actions/configure-pages@v4
+
+    - name: Upload artifact
+      uses: actions/upload-pages-artifact@v3
+      with:
+        path: docs/_build/html
+
+    - name: Deploy to GitHub Pages
+      id: deployment
+      uses: actions/deploy-pages@v4

trustwise-0.1.0a1/.github/workflows/static.yml

@@ -0,0 +1,43 @@
+# Simple workflow for deploying static content to GitHub Pages
+name: Deploy static content to Pages
+
+on:
+  # Runs on pushes targeting the default branch
+  push:
+    branches: ["main"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  # Single deploy job since we're just deploying
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          # Upload entire repository
+          path: '.'
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

trustwise-0.1.0a1/.gitignore

@@ -0,0 +1,174 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc

trustwise-0.1.0a1/PKG-INFO

@@ -0,0 +1,268 @@
+Metadata-Version: 2.4
+Name: trustwise
+Version: 0.1.0a1
+Summary: Trustwise SDK for evaluating AI-generated content
+Project-URL: Homepage, https://trustwise.ai
+Project-URL: Documentation, https://docs.trustwise.ai
+Project-URL: Repository, https://github.com/trustwise/trustwise-sdk
+Project-URL: Issues, https://github.com/trustwise/trustwise-sdk/issues
+Author-email: Trustwise Team <team@trustwise.ai>
+License-Expression: MIT
+Keywords: ai,alignment,evaluation,metrics,safety
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.11
+Requires-Dist: importlib-metadata>=6.0.0; python_version < '3.8'
+Requires-Dist: pydantic>=2.11.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: urllib3>=2.4.0
+Provides-Extra: dev
+Requires-Dist: docutils>=0.21.0; extra == 'dev'
+Requires-Dist: mypy>=1.15.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest-mock>=3.10.0; extra == 'dev'
+Requires-Dist: pytest>=7.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.11.0; extra == 'dev'
+Requires-Dist: sphinx-autodoc-typehints>=2.0.0; extra == 'dev'
+Requires-Dist: sphinx-rtd-theme>=3.0.2; extra == 'dev'
+Requires-Dist: sphinx>=8.2.0; extra == 'dev'
+Requires-Dist: tox>=4.0.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# 🦉 Trustwise Python SDK
+
+A powerful, flexible SDK for evaluating AI-generated content with Trustwise's Safety and Alignment metrics. This SDK provides a clean path-based versioning approach that makes working with different API versions intuitive and explicit.
+
+## 🔥 Features
+
+- **Path-based API versioning**: Access specific API versions directly using intuitive paths (e.g., `sdk.safety.v3.faithfulness`)
+- **Backward compatibility**: Maintain code simplicity with default version methods (e.g., `sdk.safety.faithfulness`)
+- **Comprehensive evaluation metrics**: Assess faithfulness, relevancy, clarity, helpfulness, and more
+- **LLM guardrails**: Set thresholds and automatically validate responses against multiple metrics
+- **Framework integrations**: Ready-to-use callback handlers for LangChain and LlamaIndex
+- **Detailed analytics**: Get comprehensive insights from every evaluation, including scores, facts, and analysis
+
+## 📦 Installation
+
+```bash
+pip install trustwise
+```
+
+## 🚀 Quick Start
+
+Get started with Trustwise in just a few lines of code:
+
+```python
+import os
+from trustwise.sdk import TrustwiseSDK
+from trustwise.sdk.config import TrustwiseConfig
+
+# Set your API key
+os.environ["TW_API_KEY"] = "your-api-key"
+
+# Initialize the SDK
+config = TrustwiseConfig()
+trustwise = TrustwiseSDK(config)
+```
+
+### 🛡️ Safety Metrics
+
+Evaluate content safety and faithfulness:
+
+```python
+# Example context
+context = [{
+    "node_text": "Paris is the capital of France.",
+    "node_score": 0.95,
+    "node_id": "doc:idx:1"
+}]
+
+# Evaluate faithfulness
+result = trustwise.safety.v3.faithfulness.evaluate(
+    query="What is the capital of France?",
+    response="The capital of France is Paris.",
+    context=context
+)
+print(f"Faithfulness score: {result['score']}")
+
+# Evaluate PII detection
+pii_result = trustwise.safety.v3.pii.evaluate(
+    text="My email is john@example.com and my phone is 123-456-7890",
+    allowlist=["john@example.com"]  # Allow specific PII
+)
+print(f"PII detection result: {pii_result}")
+```
+
+### 🎯 Alignment Metrics
+
+Assess content quality and alignment:
+
+```python
+# Evaluate clarity
+clarity_result = trustwise.alignment.v1.clarity.evaluate(
+    query="What is the capital of France?",
+    response="The capital of France is Paris."
+)
+print(f"Clarity score: {clarity_result['score']}")
+
+# Evaluate tone
+tone_result = trustwise.alignment.v1.tone.evaluate(
+    response="The capital of France is Paris."
+)
+print(f"Tone result: {tone_result}")
+```
+
+### 💰 Performance Metrics
+
+Evaluate cost and carbon emissions for your AI models:
+
+```python
+# Evaluate cost for OpenAI model
+cost_result = trustwise.performance.v1.cost(
+    model_name="gpt-3.5-turbo",
+    model_type="LLM",
+    model_provider="OpenAI",
+    number_of_queries=5,
+    total_prompt_tokens=950,
+    total_completion_tokens=50
+)
+print(f"Cost per run: ${cost_result['cost_estimate_per_run']}")
+print(f"Total project cost: ${cost_result['total_project_cost_estimate']}")
+
+# Evaluate carbon emissions
+carbon_result = trustwise.performance.v1.carbon(
+    processor_name="NVIDIA A100",
+    provider_name="AWS",
+    provider_region="us-east-1",
+    instance_type="p4d.24xlarge",
+    average_latency=100
+)
+print(f"Carbon emissions: {carbon_result['carbon_emissions']} kg CO2e")
+```
+
+### 🚧 Guardrails
+
+Create guardrails to automatically validate responses:
+
+```python
+# Create a multi-metric guardrail
+guardrail = trustwise.guardrails(
+    thresholds={
+        "faithfulness": 0.8,
+        "answer_relevancy": 0.7,
+        "clarity": 0.7
+    },
+    block_on_failure=True
+)
+
+# Evaluate with multiple metrics
+evaluation = guardrail.evaluate_response(
+    query="What is the capital of France?",
+    response="The capital of France is Paris.",
+    context=context
+)
+
+print("Guardrail Evaluation:")
+print(f"Passed all checks: {evaluation['passed']}")
+print(f"Response blocked: {evaluation['blocked']}")
+for metric, result in evaluation['results'].items():
+    print(f" - {metric}: {result['passed']} (score: {result['result'].get('score')})")
+```
+
+## 🔐 API Key Setup
+
+Get your API Key by logging in through Github -> [link](https://trustwise.ai)
+
+The SDK requires an API key to authenticate requests. You can provide the API key in several ways:
+
+```python
+# Option 1: From environment variable (recommended)
+import os
+from trustwise.sdk import TrustwiseSDK
+from trustwise.sdk.config import TrustwiseConfig
+
+# Set environment variable (you can also set this in your shell or .env file)
+os.environ["TW_API_KEY"] = "your-api-key"
+
+# Initialize with configuration object
+config = TrustwiseConfig()  # Will automatically use TW_API_KEY from environment
+trustwise = TrustwiseSDK(config)
+
+# Option 2: Direct initialization
+config = TrustwiseConfig(api_key="your-api-key")
+trustwise = TrustwiseSDK(config)
+
+# Option 3: Mixed approach (direct values take precedence)
+os.environ["TW_API_KEY"] = "env-key"  # Set environment variable
+config = TrustwiseConfig(api_key="direct-key")  # Will override TW_API_KEY
+trustwise = TrustwiseSDK(config)
+```
+
+## 📚 API Versioning
+
+The SDK supports both explicit version paths and default version usage:
+
+```python
+# Using explicit version path
+result = trustwise.safety.v3.faithfulness.evaluate(...)
+
+# Using default version (backward compatibility)
+result = trustwise.safety.faithfulness.evaluate(...)
+
+# Set default version explicitly
+trustwise.safety.set_version("v3")
+```
+
+## 📊 Available Metrics
+
+### Safety Metrics (v3)
+- Faithfulness
+- Answer Relevancy
+- Context Relevancy
+- Summarization
+- PII Detection
+- Prompt Injection Detection
+
+### Alignment Metrics (v1)
+- Clarity
+- Helpfulness
+- Toxicity
+- Tone
+- Formality
+- Simplicity
+- Sensitivity
+
+### Performance Metrics (v1)
+- Cost Estimation
+  - Support for multiple providers (OpenAI, Hugging Face, Azure)
+  - Support for both LLM and Reranker models
+  - Detailed cost breakdown per run and total project cost
+- Carbon Emissions
+  - Processor-specific emissions calculation
+  - Provider and region-aware estimates
+  - Latency-based calculations
+
+## 📝 License
+
+This project is licensed under the MIT License - see the LICENSE file for details.
+
+## 🤝 Contributing
+
+We welcome contributions! If you find a bug, have a feature request, or want to contribute code, please create an issue or submit a pull request.
+
+### Git Hooks
+
+This repository includes git hooks to ensure code quality. To install them:
+
+```bash
+# Make sure you're in the repository root
+./scripts/install-hooks.sh
+```
+
+The hooks will run tests, linting, and documentation checks before each push to ensure code quality.