model-library 0.1.0__tar.gz

model_library-0.1.0/.gitattributes

@@ -0,0 +1,4 @@
+* text=auto
+*.json text eol=lf
+*.yaml text eol=lf
+*.yml text eol=lf

model_library-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,96 @@
+name: Publish vals-model-library to PyPI
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  build:
+    name: Build Wheel Distribution
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build tools
+        run: python3 -m pip install build --user
+
+      - name: Build a binary wheel and a source tarball
+        run: python3 -m build
+
+      - name: Store the distribution packages
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+  publish-to-pypi:
+    name: Publish to Production PyPi Server
+    if: startsWith(github.ref, 'refs/tags/v')
+    needs:
+      - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi-prod
+      url: https://pypi.org/p/vals-model-library
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Download all the dists
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+      - name: Publish distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: Sign Distribution and Upload to GitHub Release
+    needs:
+      - publish-to-pypi
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+      id-token: write
+
+    steps:
+      - name: Download all the dists
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+      - name: Sign the dists with Sigstore
+        uses: sigstore/gh-action-sigstore-python@v2.1.1
+        with:
+          inputs: |
+            ./dist/*.tar.gz
+            ./dist/*.whl
+
+      - name: Create GitHub Release
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >-
+          gh release create
+          '${{ github.ref_name }}'
+          --repo '${{ github.repository }}'
+          --notes ""
+
+      - name: Upload artifact signatures to GitHub Release
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >-
+          gh release upload
+          '${{ github.ref_name }}' dist/**
+          --repo '${{ github.repository }}'

model_library-0.1.0/.github/workflows/style.yaml

@@ -0,0 +1,27 @@
+name: Style
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+jobs:
+  style:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          cache-local-path: ".setup-uv-cache"
+          cache-dependency-glob: "uv.lock"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: make install
+
+      - name: Run style check
+        run: make style-check

model_library-0.1.0/.github/workflows/test.yaml

@@ -0,0 +1,27 @@
+name: Test
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          cache-local-path: ".setup-uv-cache"
+          cache-dependency-glob: "uv.lock"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: make install
+
+      - name: Run tests
+        run: make test

model_library-0.1.0/.github/workflows/typecheck.yml

@@ -0,0 +1,27 @@
+name: Typecheck
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+jobs:
+  typecheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          cache-local-path: ".setup-uv-cache"
+          cache-dependency-glob: "uv.lock"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: make install
+
+      - name: Run typecheck
+        run: make typecheck

model_library-0.1.0/.gitignore

@@ -0,0 +1,4 @@
+model_library.egg-info
+**/__pycache__
+CLAUDE.md
+.env

model_library-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Vals AI, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

model_library-0.1.0/Makefile

@@ -0,0 +1,90 @@
+.PHONY: help install test test-integration test-all style style-check typecheck config run-models examples browse_models
+
+help:
+	@echo "Makefile for model-library"
+	@echo "Usage:"
+	@echo "  make install          Install dependencies"
+	@echo "  make test             Run unit tests"
+	@echo "  make test-integration Run integration tests (requires API keys)"
+	@echo "  make test-all         Run all tests (unit + integration)"
+	@echo "  make style            Lint & Format"
+	@echo "  make style-check      Check style"
+	@echo "  make typecheck        Typecheck"
+	@echo "  make config           Generate all_models.json"
+	@echo "  make run-models       Run all models"
+	@echo "  make examples         Run all examples"
+	@echo "  make examples <model> Run all examples with specified model"
+	@echo "  make browse_models    Interactively browse models and their configurations"
+
+install:
+	uv venv
+	uv sync --dev
+	@echo "🎉 Done! Run 'source .venv/bin/activate' to activate the environment locally."
+
+venv_check:
+	@if [ ! -f .venv/bin/activate ]; then \
+		echo "❌ Virtualenv not found! Run \`make install\` first."; \
+		exit 1; \
+	fi
+
+test: venv_check
+	@echo "Running unit tests..."
+	@uv run pytest tests/unit/ -m "not integration"
+test-integration: venv_check
+	@echo "Running integration tests (requires API keys)..."
+	@uv run pytest tests/integration/ -m "not unit"
+test-all: venv_check
+	@echo "Running all tests..."
+	@uv run pytest
+
+format: venv_check
+	@uv run ruff format .
+lint: venv_check
+	@uv run ruff check --fix .
+style: format lint
+
+style-check: venv_check
+	@uv run ruff format --check .
+	@uv run ruff check .
+
+typecheck: venv_check
+	@uv run basedpyright
+
+config: venv_check
+	@uv run python scripts/config.py
+
+run-models: venv_check
+	@uv run python -m scripts.run_models
+
+examples: venv_check
+	@echo "Running examples with model: $(filter-out $@,$(MAKECMDGOALS))"
+	@echo "\n=== Running basics.py ==="
+	@uv run python -m examples.basics $(filter-out $@,$(MAKECMDGOALS)) || true
+	@echo "\n=== Running images.py ==="
+	@uv run python -m examples.images $(filter-out $@,$(MAKECMDGOALS)) || true
+	@echo "\n=== Running files.py ==="
+	@uv run python -m examples.files $(filter-out $@,$(MAKECMDGOALS)) || true
+	@echo "\n=== Running tool_calls.py ==="
+	@uv run python -m examples.tool_calls $(filter-out $@,$(MAKECMDGOALS)) || true
+	@echo "\n=== Running advanced.structured_output ==="
+	@uv run python -m examples.advanced.structured_output $(filter-out $@,$(MAKECMDGOALS)) || true
+	@echo "\n=== Running advanced.batch ==="
+	@uv run python -m examples.advanced.batch $(filter-out $@,$(MAKECMDGOALS)) || true
+	@echo "\n=== Running advanced.custom_retrier ==="
+	@uv run python -m examples.advanced.custom_retrier $(filter-out $@,$(MAKECMDGOALS)) || true
+	@echo "\n=== Running advanced.stress ==="
+	@uv run python -m examples.advanced.stress $(filter-out $@,$(MAKECMDGOALS)) || true
+	
+	@if [ -z "$(filter-out $@,$(MAKECMDGOALS))" ]; then \
+		echo "\n=== Running embeddings.py ==="; \
+		uv run python -m examples.embeddings; \
+
+		echo "\n=== Running advanced.deep_research ==="; \
+		uv run python -m examples.advanced.deep_research; \
+	fi
+
+	@echo "\n✅ All examples completed!"
+	
+browse_models: venv_check
+	@uv run python -m scripts.explore_models
+

model_library-0.1.0/PKG-INFO

@@ -0,0 +1,268 @@
+Metadata-Version: 2.4
+Name: model-library
+Version: 0.1.0
+Summary: Model Library for vals.ai 
+Author-email: "Vals AI, Inc." <contact@vals.ai>
+License: MIT
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: typing-extensions<5.0,>=4.14.1
+Requires-Dist: pydantic<3.0,>=2.11.7
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: backoff<3.0,>=2.2.1
+Requires-Dist: redis<7.0,>=6.2.0
+Requires-Dist: tiktoken==0.11.0
+Requires-Dist: pillow
+Requires-Dist: openai<2.0,>=1.97.1
+Requires-Dist: anthropic<1.0,>=0.57.1
+Requires-Dist: together<2.0,>=1.5.25
+Requires-Dist: mistralai<2.0,>=1.9.10
+Requires-Dist: xai-sdk<2.0,>=1.0.0
+Requires-Dist: ai21<5.0,>=4.0.3
+Requires-Dist: boto3<2.0,>=1.38.27
+Requires-Dist: google-genai[aiohttp]<2.0,>=1.48.0
+Requires-Dist: google-cloud-storage>=1.26.0
+Dynamic: license-file
+
+# Model Library
+
+Open-source model library for interacting with a variety of LLM providers. Originally developed for internal use at [vals.ai](https://vals.ai/) benchmarks. This tool is designed to be a general-purpose solution for any project requiring a unified interface for multiple model providers.
+
+`pip install model-library`
+
+**Note**: This library is undergoing rapid development. Expect breaking changes.
+
+## Features
+
+### Providers
+
+- AI21 Labs
+- Alibaba
+- Amazon Bedrock
+- Anthropic
+- Azure OpenAI
+- Cohere
+- DeepSeek
+- Fireworks
+- Google Gemini
+- Mistral
+- Perplexity
+- Together AI
+- OpenAI
+- X AI
+- ZhipuAI (zai)
+
+Run `python -m scripts.browse_models` to browse the model registry.
+
+### Supported Input
+
+- Images
+- Files
+- Tools (with full history)
+- Batch
+- Reasoning
+- Custom Parameters
+
+## Usage
+
+Here is a basic example of how to query a model:
+
+```python
+import asyncio
+from model_library.registry_utils import get_registry_model
+
+async def main():
+    # Load a model from the registry
+    model = get_registry_model("openai/gemini-2.5-flash")
+
+    # Query the model with a simple text input
+    response = await model.query("What is QSBS? Explain your thinking in detail and make it concise.")
+
+    # Logger automatically logs the response
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+The model registry holds model attributes, ex. reasoning, file support, tool support, max tokens. You may also use models not included in the registry.
+
+```python
+model = get_raw_model("openai/gpt-3.5-turbo", config=LLMConfig(max_tokens=1000))
+```
+
+### Environment Setup
+
+The model library will use:
+- Environment varibles for API keys
+    - OPENAI_API_KEY
+    - ANTHROPIC_API_KEY
+    - GOOGLE_API_KEY
+    - ...
+    
+- Variables set through model_library.settings
+```python
+from model_library import model_library_settings
+
+model_library_settings.set(MY_KEY="my-key")
+```
+
+
+### System Prompt
+
+```bash
+python -m examples.basics
+```
+
+```python
+await model.query(
+    [TextInput(text="Hello, how are you?")],
+    system_prompt="You are a pirate, answer in the speaking style of a pirate. Keeps responses under 10 words",
+)
+```
+
+### Image/File Input
+
+Supports base64, url, and file id (file upload)
+
+```bash
+python -m examples.images
+```
+
+```python
+red_image_content = b"..."
+
+await model.query(
+    [
+        TextInput(text="What color is the image?"),
+        FileWithBase64(
+            type="image",
+            name="red_image.png",
+            mime="png",
+            base64=base64.b64encode(red_image_content).decode("utf-8"),
+        ),
+    ]
+)
+```
+
+### Tool Calls
+
+```bash
+python -m examples.tool_calls
+```
+
+```python
+tools = [
+    ToolDefinition(
+        name="get_weather",
+        body=ToolBody(
+            name="get_weather",
+            description="Get current temperature in a given location",
+            properties={
+                "location": {
+                    "type": "string",
+                    "description": "City and country e.g. Bogotá, Colombia",
+                },
+            },
+            required=["location"],
+        ),
+    )
+]
+
+output1 = await model.query(
+    [TextInput(text="What is the weather in SF right now?")],
+    tools=tools,
+)
+
+output2 = await model.query(
+    [
+        # assume one tool call was made
+        ToolResult(tool_call=output1.tool_calls[0], result="25C"),
+        TextInput(
+            text="Also, includes some weird emojies in your answer (at least 8 of them)"
+        ),
+    ],
+    history=output1.history,
+    tools=tools,
+```
+
+### Full examples
+
+You can run `make examples` (default models) or `make example <model>` to run all examples.
+
+`python -m examples.basics`
+
+`python -m examples.images`
+
+`python -m examples.files`
+
+`python -m examples.tool_calls`
+
+`python -m examples.embeddings`
+
+`python -m examples.advanced.batch`
+
+`python -m examples.advanced.custom_retrier`
+
+`python -m examples.advanced.stress`
+
+`python -m examples.advanced.deep_research`
+
+
+## Architecture
+
+Designed to abstract different LLM providers:
+
+- **LLM Base Class**: An abstract base class that defines a common interface for all models
+- **Model Registry**: A central registry that loads model configurations from YAML files
+- **Provider-Specific Implementations**: Concrete classes for each provider (e.g., OpenAI, Google, Anthropic) that inherit from the `LLM` base class
+- **Data Models**: A set of `pydantic` models for representing various input and output types, such as `TextInput`, `FileWithBase64`, `ToolDefinition`, and `ToolResult`. This ensures code is model agnostic, and easy to maintain.
+- **Retry Logic**: A set of retry strategies for handling errors and rate limiting
+
+## Contributing
+
+### Setup
+
+We use [uv](https://docs.astral.sh/uv/getting-started/installation/) for dependency management.
+A Makefile is provided to help with development.
+
+To install dependencies, run:
+
+```bash
+make install
+```
+
+### Makefile commands
+
+```bash
+make install          Install dependencies"
+make test             Run unit tests"
+make test-integration Run integration tests (requires API keys)"
+make test-all         Run all tests (unit + integration)"
+make style            Lint & Format"
+make style-check      Check style"
+make typecheck        Typecheck"
+make config           Generate all_models.json"
+make run-models       Run all models"
+make examples         Run all examples"
+make examples <model> Run all examples with specified model"
+make browse-models    Browse all models"
+```
+
+### Testing
+
+#### Unit Tests
+
+Unit tests do not require API keys
+
+```bash
+make test-unit
+```
+
+#### Integration Tests
+
+Make sure you have API keys configured
+
+```bash
+make test-integration
+```