caniollama 0.1.0__tar.gz

caniollama-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,77 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "latest"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync --all-extras --dev
+
+      - name: Lint
+        run: uv run ruff check .
+
+      - name: Type check
+        run: uv run mypy src
+
+      - name: Test
+        run: uv run pytest -v
+
+  suite-version-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check SUITE_VERSION bump if cases.py changed
+        run: |
+          # Skip on main branch (no base to compare against)
+          if [ "${{ github.ref }}" = "refs/heads/main" ]; then
+            echo "Running on main branch, skipping check"
+            exit 0
+          fi
+
+          # For PRs, compare against the base branch
+          BASE_REF="${{ github.base_ref || 'main' }}"
+          git fetch origin "$BASE_REF"
+
+          # Check if cases.py was modified
+          if git diff --name-only "origin/$BASE_REF"...HEAD | grep -q "src/compat_check/suite/cases.py"; then
+            echo "cases.py was modified, checking if SUITE_VERSION was bumped..."
+
+            # Check if version.py was also modified
+            if ! git diff --name-only "origin/$BASE_REF"...HEAD | grep -q "src/compat_check/suite/version.py"; then
+              echo "ERROR: cases.py was modified but version.py was not updated"
+              echo "Please bump SUITE_VERSION in src/compat_check/suite/version.py"
+              exit 1
+            fi
+
+            # Check if SUITE_VERSION value actually changed
+            if ! git diff "origin/$BASE_REF"...HEAD src/compat_check/suite/version.py | grep -q "^[+-]SUITE_VERSION"; then
+              echo "ERROR: version.py was modified but SUITE_VERSION value was not changed"
+              echo "Please bump SUITE_VERSION in src/compat_check/suite/version.py"
+              exit 1
+            fi
+
+            echo "✓ SUITE_VERSION was properly bumped"
+          else
+            echo "cases.py was not modified, no version bump required"
+          fi

caniollama-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,42 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "latest"
+
+      - name: Build package
+        run: uv build
+
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # required for PyPI trusted publishing (OIDC), no stored API token
+    steps:
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

caniollama-0.1.0/.gitignore

@@ -0,0 +1,60 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+
+# uv
+uv.lock
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+
+# Linting
+.ruff_cache/
+.mypy_cache/
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Data files (except templates)
+compat-registry/data/results/*/
+compat-registry/data/*.db
+compat-registry/data/*.db-journal
+
+# Build artifacts
+*.whl
+*.tar.gz

caniollama-0.1.0/CHANGELOG.md

@@ -0,0 +1,7 @@
+# Changelog
+
+## [0.1.0] - Unreleased
+### Added
+- Initial CLI: `caniollama check [model]`
+- Fixed test suite v1.0.0: single tool, 5-tool selection, no-tool control, multi-turn tool result
+- Opt-in, metadata-only submission to the registry

caniollama-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 caniollama contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

caniollama-0.1.0/PKG-INFO

@@ -0,0 +1,75 @@
+Metadata-Version: 2.4
+Name: caniollama
+Version: 0.1.0
+Summary: Standardized compatibility checker for local Ollama model tool-calling support
+Project-URL: Homepage, https://github.com/AnjalLLL/caniollama
+Project-URL: Repository, https://github.com/AnjalLLL/caniollama
+Project-URL: Issues, https://github.com/AnjalLLL/caniollama/issues
+Author: caniollama contributors
+License: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: pydantic>=2.6
+Requires-Dist: rich>=13.7
+Requires-Dist: typer>=0.12
+Description-Content-Type: text/markdown
+
+# caniollama
+
+Does this local Ollama model actually support tool calling? Find out in seconds, not by
+reading a six-month-old blog post.
+
+```bash
+uvx caniollama check qwen2.5-coder:7b
+```
+
+or test everything you've pulled:
+
+```bash
+uvx caniollama check
+```
+
+## What it does
+
+Runs a small, fixed test suite against a locally running Ollama instance and classifies
+the result into one of three real answers:
+
+1. **Structured pass** -- the model returns a proper `tool_calls` field.
+2. **JSON-in-content fallback** -- the model knows tool calling exists but emits the
+   intent as JSON-shaped text instead of the structured field.
+3. **Rejected by backend** -- Ollama 400s the moment `tools` is included.
+
+After a run, you'll be asked -- every time, explicitly -- whether to submit the result
+(metadata only: model name, quantization, Ollama version, pass/fail per case) to the
+public [caniollama registry](https://github.com/caniollama/caniollama-registry). See
+[`docs/privacy.md`](docs/privacy.md) for exactly what is and isn't ever sent.
+
+## Install
+
+No install needed for a one-off check:
+
+```bash
+uvx caniollama check <model>
+# or
+pipx run caniollama check <model>
+```
+
+For repeated use:
+
+```bash
+uv tool install caniollama
+```
+
+## Development
+
+```bash
+uv sync
+uv run pytest
+uv run ruff check .
+uv run mypy src
+```
+
+## License
+
+MIT

caniollama-0.1.0/README.md

@@ -0,0 +1,58 @@
+# caniollama
+
+Does this local Ollama model actually support tool calling? Find out in seconds, not by
+reading a six-month-old blog post.
+
+```bash
+uvx caniollama check qwen2.5-coder:7b
+```
+
+or test everything you've pulled:
+
+```bash
+uvx caniollama check
+```
+
+## What it does
+
+Runs a small, fixed test suite against a locally running Ollama instance and classifies
+the result into one of three real answers:
+
+1. **Structured pass** -- the model returns a proper `tool_calls` field.
+2. **JSON-in-content fallback** -- the model knows tool calling exists but emits the
+   intent as JSON-shaped text instead of the structured field.
+3. **Rejected by backend** -- Ollama 400s the moment `tools` is included.
+
+After a run, you'll be asked -- every time, explicitly -- whether to submit the result
+(metadata only: model name, quantization, Ollama version, pass/fail per case) to the
+public [caniollama registry](https://github.com/caniollama/caniollama-registry). See
+[`docs/privacy.md`](docs/privacy.md) for exactly what is and isn't ever sent.
+
+## Install
+
+No install needed for a one-off check:
+
+```bash
+uvx caniollama check <model>
+# or
+pipx run caniollama check <model>
+```
+
+For repeated use:
+
+```bash
+uv tool install caniollama
+```
+
+## Development
+
+```bash
+uv sync
+uv run pytest
+uv run ruff check .
+uv run mypy src
+```
+
+## License
+
+MIT

caniollama-0.1.0/docs/privacy.md

@@ -0,0 +1,27 @@
+# What caniollama sends, and what it never sends
+
+## Sent, only if you say yes, every single run
+
+- Model name (e.g. `qwen2.5-coder:7b`)
+- Quantization level (e.g. `q4_K_M`), if Ollama reports one
+- Ollama version
+- Test suite version
+- Coarse OS (`linux` / `macos` / `windows`)
+- Pass/fail outcome per test case (one of: structured pass, JSON-in-content
+  fallback, rejected by backend, error/timeout)
+- Timestamp
+
+## Never sent, under any circumstances
+
+- The test prompts themselves
+- Any model output, structured or otherwise
+- Anything from your local filesystem, environment, or network beyond the above
+- Anything if you decline the prompt -- there is no "phone home anyway" path
+
+## Why so little
+
+This project measures whether the *plumbing* works (does tool calling return
+a structured field, does it fall back to text, does the backend reject the
+request) -- not how good a model's reasoning is. That narrower question needs
+far less data than a quality benchmark would, so we ask for less than we
+technically could.

caniollama-0.1.0/docs/test-suite-spec.md

@@ -0,0 +1,50 @@
+# Test suite spec — v1.0.0
+
+This documents what each fixed case in `src/compat_check/suite/cases.py` tests and why.
+Any change to a prompt, a tool schema, or a scoring rule in `scoring.py` requires bumping
+`SUITE_VERSION` in `suite/version.py` — this is what keeps registry results comparable
+over time (see `compat-registry`'s `docs/data-format.md`).
+
+## `single_tool_basic`
+One relevant tool offered, one unambiguous prompt ("What is the weather in Tokyo?").
+This is the baseline: does the model return a structured `tool_calls` field at all,
+under the easiest possible conditions. If a model fails this one, it almost certainly
+falls into compatibility category 2 or 3.
+
+## `multi_tool_5x`
+Same prompt shape, but five tools offered (one relevant, four irrelevant no-ops). Tests
+whether structured calling holds up once the model has to *select* the right tool from
+a small list, rather than just echoing the only option it was given.
+
+## `no_tools_needed_control`
+A prompt with no tool relevance at all ("Say hello in French"), with a weather tool
+still offered. This is a control: a model that calls a tool here anyway isn't
+necessarily marked as failing (over-eager tool use is a reasoning-quality concern, out
+of scope for this project), but it's tracked separately so the registry can eventually
+surface "tends to over-call tools" as its own signal if that becomes useful.
+
+## `multiturn_tool_result`
+First turn asks for weather; the suite then injects a synthetic tool result message and
+asks again. Tests whether the model can continue a conversation that already contains a
+completed tool call/result correctly, rather than looping, re-calling the same tool, or
+breaking format. This is the case most likely to expose category 2 behavior in models
+that pass `single_tool_basic` — first-turn tool calling is the easy case; handling the
+result correctly is where a lot of "sort of supports tools" models fall down.
+
+## Deliberately out of scope
+
+- Tool call **argument correctness** (e.g. did it extract "Tokyo" correctly) — that's
+  reasoning quality, not plumbing, and belongs to a benchmark project, not this one.
+- Parallel/simultaneous tool calls — a plausible v1.1.0 addition once the four cases
+  above are validated against enough real models.
+- Streaming responses (`stream: true`) — Ollama's tool-calling behavior under streaming
+  differs from non-streaming in ways worth a dedicated case, deferred to a future
+  suite version.
+
+## Adding a new case
+
+1. Add the `TestCase` to `cases.py`.
+2. Add scoring logic to `scoring.py` if the existing pass/fail heuristics don't cover it.
+3. Document it here.
+4. Bump `SUITE_VERSION`.
+5. Add a fixture-based test in `tests/test_scoring.py`.

caniollama-0.1.0/pyproject.toml

@@ -0,0 +1,39 @@
+[project]
+name = "caniollama"
+version = "0.1.0"
+description = "Standardized compatibility checker for local Ollama model tool-calling support"
+readme = "README.md"
+license = { text = "MIT" }
+requires-python = ">=3.10"
+authors = [{ name = "caniollama contributors" }]
+dependencies = [
+    "httpx>=0.27",
+    "pydantic>=2.6",
+    "typer>=0.12",
+    "rich>=13.7",
+]
+
+[project.scripts]
+caniollama = "compat_check.cli:app"
+
+[project.urls]
+Homepage = "https://github.com/AnjalLLL/caniollama"
+Repository = "https://github.com/AnjalLLL/caniollama"
+Issues = "https://github.com/AnjalLLL/caniollama/issues"
+
+[dependency-groups]
+dev = ["pytest>=8.0", "pytest-asyncio>=0.23", "ruff>=0.5", "mypy>=1.10", "respx>=0.21"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/compat_check"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"

caniollama-0.1.0/src/compat_check/__init__.py

@@ -0,0 +1,3 @@
+"""caniollama - does this local Ollama model actually support tool calling?"""
+
+__version__ = "0.1.0"

caniollama-0.1.0/src/compat_check/__main__.py

@@ -0,0 +1,4 @@
+from compat_check.cli import app
+
+if __name__ == "__main__":
+    app()

caniollama-0.1.0/src/compat_check/cli.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import asyncio
+
+import typer
+from rich.console import Console
+
+from compat_check import __version__
+from compat_check.config import settings
+from compat_check.discovery import current_os, discover_models, discover_ollama_version
+from compat_check.models import RunReport
+from compat_check.ollama_client import OllamaClient
+from compat_check.report import render_report
+from compat_check.submission import prompt_for_submission_consent, submit_result
+from compat_check.suite.runner import run_suite
+from compat_check.suite.version import SUITE_VERSION
+
+app = typer.Typer(
+    name="caniollama",
+    help="Does this local Ollama model actually support tool calling? Find out in seconds.",
+    no_args_is_help=False,
+)
+console = Console()
+
+
+@app.command()
+def version() -> None:
+    """Show the package and test-suite versions."""
+    console.print(f"caniollama {__version__} (test suite {SUITE_VERSION})")
+
+
+@app.command()
+def check(
+    model: str = typer.Argument(
+        None, help="Model to test, e.g. 'qwen2.5-coder:7b'. Omit to test all pulled models."
+    ),
+    ollama_url: str = typer.Option(settings.ollama_base_url, help="Ollama base URL."),
+    submit: bool = typer.Option(
+        None,
+        "--submit/--no-submit",
+        help="Skip the interactive prompt: always/never submit. Default is to ask.",
+    ),
+    registry_url: str = typer.Option(settings.registry_base_url, help="Registry base URL."),
+) -> None:
+    """Run the compatibility test suite against a local Ollama model (or all pulled models)."""
+    asyncio.run(_check_async(model, ollama_url, submit, registry_url))
+
+
+async def _check_async(
+    model: str | None, ollama_url: str, submit_flag: bool | None, registry_url: str
+) -> None:
+    client = OllamaClient(base_url=ollama_url)
+
+    try:
+        ollama_version = await discover_ollama_version(client)
+    except Exception as exc:
+        console.print(f"[bold red]Could not reach Ollama at {ollama_url}: {exc}[/]")
+        raise typer.Exit(code=1) from exc
+
+    targets: list[tuple[str, str | None]]
+    if model:
+        targets = [(model, None)]
+    else:
+        discovered = await discover_models(client)
+        if not discovered:
+            console.print("[yellow]No locally pulled models found.[/]")
+            raise typer.Exit(code=0)
+        targets = [(m.name, m.quantization) for m in discovered]
+
+    for target_name, target_quantization in targets:
+        console.print(f"[dim]Running suite against {target_name}...[/]")
+        results = await run_suite(client, target_name)
+        report = RunReport(
+            model_name=target_name,
+            model_quantization=target_quantization,
+            ollama_version=ollama_version,
+            suite_version=SUITE_VERSION,
+            os_name=current_os(),
+            results=results,
+        )
+        render_report(report, console)
+
+        should_submit = submit_flag
+        if should_submit is None:
+            should_submit = prompt_for_submission_consent(console)
+
+        if should_submit:
+            ok = await submit_result(report, base_url=registry_url)
+            if ok:
+                console.print("[green]Submitted. Thanks for contributing to the registry![/]\n")
+            else:
+                console.print("[yellow]Submission failed -- registry may be unreachable.[/]\n")
+        else:
+            console.print("[dim]Not submitted.[/]\n")
+
+
+if __name__ == "__main__":
+    app()

caniollama-0.1.0/src/compat_check/config.py

@@ -0,0 +1,19 @@
+"""Runtime configuration, resolved from environment variables with sane defaults."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Settings:
+    ollama_base_url: str = os.environ.get("OLLAMA_BASE_URL", "http://localhost:11434")
+    registry_base_url: str = os.environ.get(
+        "CANIOLLAMA_REGISTRY_URL", "https://registry.caniollama.dev"
+    )
+    request_timeout_seconds: float = float(os.environ.get("CANIOLLAMA_TIMEOUT", "30"))
+    submit_timeout_seconds: float = float(os.environ.get("CANIOLLAMA_SUBMIT_TIMEOUT", "10"))
+
+
+settings = Settings()

caniollama-0.1.0/src/compat_check/discovery.py

@@ -0,0 +1,37 @@
+"""Finds locally pulled Ollama models and the current Ollama/OS metadata."""
+
+from __future__ import annotations
+
+import platform
+from dataclasses import dataclass
+
+from compat_check.ollama_client import OllamaClient
+
+
+@dataclass
+class DiscoveredModel:
+    name: str            # e.g. "qwen2.5-coder:7b"
+    quantization: str | None
+
+
+def _guess_quantization(model_tag: dict) -> str | None:
+    details = model_tag.get("details") or {}
+    return details.get("quantization_level")
+
+
+async def discover_models(client: OllamaClient) -> list[DiscoveredModel]:
+    raw = await client.list_models()
+    return [
+        DiscoveredModel(name=m["name"], quantization=_guess_quantization(m))
+        for m in raw
+        if "name" in m
+    ]
+
+
+async def discover_ollama_version(client: OllamaClient) -> str:
+    return await client.version()
+
+
+def current_os() -> str:
+    system = platform.system().lower()
+    return {"darwin": "macos", "linux": "linux", "windows": "windows"}.get(system, system)

caniollama-0.1.0/src/compat_check/models.py

@@ -0,0 +1,68 @@
+"""Data shapes used across the CLI.
+
+`SubmissionPayload` is deliberately the *only* model allowed to reach
+`submission.py`'s network call. Every field is a bounded type (str version
+tag, enum, datetime) -- none of them are capable of carrying a prompt or a
+raw model output. That's a structural guarantee, not just a convention;
+`tests/test_submission.py` asserts it via schema introspection.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from enum import Enum
+
+from pydantic import BaseModel, Field
+
+
+class CaseOutcome(str, Enum):
+    """The three real-world answers to 'does tool calling work', plus a bucket for errors."""
+
+    STRUCTURED_PASS = "structured_pass"            # category 1: native tool_calls field
+    JSON_IN_CONTENT_FALLBACK = "json_fallback"      # category 2: intent, but shoved into content
+    REJECTED_BY_BACKEND = "rejected"                # category 3: backend 400s on `tools`
+    TIMEOUT_OR_ERROR = "error"
+
+
+class CaseResult(BaseModel):
+    """Local-only, detailed result for a single test case. Never submitted as-is."""
+
+    case_id: str
+    outcome: CaseOutcome
+    latency_ms: int | None = None
+    detail: str | None = None  # short internal note, e.g. "HTTP 400", never model output
+
+
+class RunReport(BaseModel):
+    """Full local report for a single model run. Rich detail lives here, on disk / terminal only."""
+
+    model_name: str
+    model_quantization: str | None
+    ollama_version: str
+    suite_version: str
+    os_name: str
+    results: list[CaseResult]
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+
+    def to_submission_payload(self) -> SubmissionPayload:
+        return SubmissionPayload(
+            model_name=self.model_name,
+            model_quantization=self.model_quantization,
+            ollama_version=self.ollama_version,
+            suite_version=self.suite_version,
+            os=self.os_name,
+            case_results={r.case_id: r.outcome for r in self.results},
+            timestamp=self.timestamp,
+        )
+
+
+class SubmissionPayload(BaseModel):
+    """The ONLY thing ever sent to the registry. Metadata only -- never prompts, never outputs."""
+
+    model_name: str
+    model_quantization: str | None = None
+    ollama_version: str
+    suite_version: str
+    os: str
+    case_results: dict[str, CaseOutcome]
+    timestamp: datetime

caniollama-0.1.0/src/compat_check/ollama_client.py

@@ -0,0 +1,69 @@
+"""Thin async client around the local Ollama HTTP API. No third-party ollama SDK dependency
+by design -- keeps this package's dependency footprint minimal, per the "installs in seconds"
+requirement."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+
+from compat_check.config import settings
+
+
+class OllamaError(Exception):
+    """Raised for connection failures or unexpected non-tool-related errors."""
+
+
+class OllamaRejected(Exception):
+    """Raised when Ollama returns a 4xx specifically because `tools` was included
+    (compatibility category 3: the backend refuses the request outright)."""
+
+    def __init__(self, status_code: int, body: str):
+        self.status_code = status_code
+        self.body = body
+        super().__init__(f"Ollama rejected request with tools: HTTP {status_code}")
+
+
+class OllamaClient:
+    def __init__(self, base_url: str | None = None, timeout: float | None = None):
+        self.base_url = base_url or settings.ollama_base_url
+        self.timeout = timeout or settings.request_timeout_seconds
+
+    async def list_models(self) -> list[dict[str, Any]]:
+        async with httpx.AsyncClient(base_url=self.base_url, timeout=self.timeout) as client:
+            resp = await client.get("/api/tags")
+            resp.raise_for_status()
+            return resp.json().get("models", [])
+
+    async def version(self) -> str:
+        async with httpx.AsyncClient(base_url=self.base_url, timeout=self.timeout) as client:
+            resp = await client.get("/api/version")
+            resp.raise_for_status()
+            return resp.json().get("version", "unknown")
+
+    async def chat(
+        self,
+        model: str,
+        messages: list[dict[str, Any]],
+        tools: list[dict[str, Any]] | None = None,
+    ) -> dict[str, Any]:
+        """POST /api/chat. Raises OllamaRejected if the backend 4xx's specifically because
+        `tools` was included -- that's compatibility category 3, a signal we want to catch
+        distinctly from a generic connection error."""
+
+        payload: dict[str, Any] = {"model": model, "messages": messages, "stream": False}
+        if tools:
+            payload["tools"] = tools
+
+        async with httpx.AsyncClient(base_url=self.base_url, timeout=self.timeout) as client:
+            try:
+                resp = await client.post("/api/chat", json=payload)
+                resp.raise_for_status()
+                return resp.json()
+            except httpx.HTTPStatusError as exc:
+                if tools and 400 <= exc.response.status_code < 500:
+                    raise OllamaRejected(exc.response.status_code, exc.response.text) from exc
+                raise OllamaError(str(exc)) from exc
+            except httpx.RequestError as exc:
+                raise OllamaError(str(exc)) from exc

caniollama-0.1.0/src/compat_check/report.py

@@ -0,0 +1,44 @@
+"""Builds the local, human-readable summary of a run. This is the ONLY place
+detailed results are shown -- rich detail stays local, never leaves the machine
+except as the reduced SubmissionPayload (see models.py)."""
+
+from __future__ import annotations
+
+from rich.console import Console
+from rich.table import Table
+
+from compat_check.models import CaseOutcome, RunReport
+
+_OUTCOME_STYLE = {
+    CaseOutcome.STRUCTURED_PASS: ("[bold green]structured pass[/]", "✅"),
+    CaseOutcome.JSON_IN_CONTENT_FALLBACK: ("[yellow]json fallback[/]", "⚠️"),
+    CaseOutcome.REJECTED_BY_BACKEND: ("[bold red]rejected by backend[/]", "⛔"),
+    CaseOutcome.TIMEOUT_OR_ERROR: ("[red]error / timeout[/]", "❌"),
+}
+
+
+def render_report(report: RunReport, console: Console | None = None) -> None:
+    console = console or Console()
+
+    console.print(
+        f"\n[bold]{report.model_name}[/] "
+        f"(quant: {report.model_quantization or 'unknown'}, "
+        f"ollama {report.ollama_version}, suite {report.suite_version})\n"
+    )
+
+    table = Table(show_header=True, header_style="bold")
+    table.add_column("Case")
+    table.add_column("Outcome")
+    table.add_column("Latency")
+    table.add_column("Detail")
+
+    for result in report.results:
+        label, icon = _OUTCOME_STYLE[result.outcome]
+        latency = f"{result.latency_ms} ms" if result.latency_ms is not None else "-"
+        table.add_row(result.case_id, f"{icon} {label}", latency, result.detail or "")
+
+    console.print(table)
+
+    passed = sum(1 for r in report.results if r.outcome == CaseOutcome.STRUCTURED_PASS)
+    total = len(report.results)
+    console.print(f"\n[bold]{passed}/{total}[/] cases returned structured tool calls.\n")

caniollama-0.1.0/src/compat_check/submission.py

@@ -0,0 +1,49 @@
+"""Opt-in submission to the public registry.
+
+Hard rule: the ONLY thing constructed and sent here is a SubmissionPayload
+(see models.py), which is structurally incapable of holding a prompt or a raw
+model output. Nothing in this module should ever import or touch case
+prompts/messages/content -- if a future change makes that necessary, it is a
+sign the privacy boundary is being violated and should be reconsidered, not
+routed around.
+
+The prompt is shown every run, after results are already displayed -- never
+before, never silently skipped, never persisted as a "don't ask again"
+default. See docs/privacy.md for the full data-handling explanation shown to
+users.
+"""
+
+from __future__ import annotations
+
+import httpx
+from rich.console import Console
+
+from compat_check.config import settings
+from compat_check.models import RunReport
+
+
+def prompt_for_submission_consent(console: Console | None = None) -> bool:
+    console = console or Console()
+    console.print(
+        "\n[bold]Share this result with the public caniollama registry?[/]\n"
+        "Only this gets sent, nothing else: model name, quantization, Ollama "
+        "version, suite version, OS, and pass/fail per test case.\n"
+        "[dim]Prompts and model output are never sent. See docs/privacy.md.[/]\n"
+    )
+    answer = console.input("Submit? [y/N] ").strip().lower()
+    return answer in ("y", "yes")
+
+
+async def submit_result(report: RunReport, base_url: str | None = None) -> bool:
+    """Returns True on a confirmed, successful submission. Never raises on
+    network failure -- a failed submission should never crash a local run."""
+
+    payload = report.to_submission_payload()
+    url = f"{(base_url or settings.registry_base_url).rstrip('/')}/submissions"
+
+    try:
+        async with httpx.AsyncClient(timeout=settings.submit_timeout_seconds) as client:
+            resp = await client.post(url, json=payload.model_dump(mode="json"))
+            return resp.status_code in (200, 201)
+    except httpx.RequestError:
+        return False

caniollama-0.1.0/src/compat_check/suite/cases.py

@@ -0,0 +1,87 @@
+"""Fixed, deterministic test cases. Each targets one dimension of tool-calling
+compatibility. Prompts are intentionally simple and unambiguous -- the goal is
+to test plumbing ('does it call the tool'), not reasoning quality.
+
+Do not tweak prompt wording casually: any change here requires bumping
+SUITE_VERSION in version.py.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+_GET_WEATHER_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "get_weather",
+        "description": "Get the current weather for a city.",
+        "parameters": {
+            "type": "object",
+            "properties": {"city": {"type": "string", "description": "City name"}},
+            "required": ["city"],
+        },
+    },
+}
+
+_EXTRA_TOOLS = [
+    {
+        "type": "function",
+        "function": {
+            "name": f"noop_tool_{i}",
+            "description": f"Placeholder tool #{i}, not relevant to the prompt.",
+            "parameters": {"type": "object", "properties": {}},
+        },
+    }
+    for i in range(1, 5)
+]
+
+
+@dataclass
+class TestCase:
+    case_id: str
+    description: str
+    messages: list[dict[str, Any]]
+    tools: list[dict[str, Any]]
+    # If True, the correct behavior is to NOT call a tool -- used to catch
+    # models that over-eagerly call tools even when none are needed.
+    expects_no_tool_call: bool = False
+    multiturn_followup: dict[str, Any] | None = field(default=None)
+
+
+CASES: list[TestCase] = [
+    TestCase(
+        case_id="single_tool_basic",
+        description="One relevant tool, unambiguous prompt -- baseline structured call.",
+        messages=[{"role": "user", "content": "What is the weather in Tokyo?"}],
+        tools=[_GET_WEATHER_TOOL],
+    ),
+    TestCase(
+        case_id="multi_tool_5x",
+        description="Five tools offered, only one relevant -- tests selection under noise.",
+        messages=[{"role": "user", "content": "What is the weather in Nairobi?"}],
+        tools=[_GET_WEATHER_TOOL, *_EXTRA_TOOLS],
+    ),
+    TestCase(
+        case_id="no_tools_needed_control",
+        description="No tool is relevant -- control case; a good model should NOT call one.",
+        messages=[{"role": "user", "content": "Say hello in French."}],
+        tools=[_GET_WEATHER_TOOL],
+        expects_no_tool_call=True,
+    ),
+    TestCase(
+        case_id="multiturn_tool_result",
+        description=(
+            "Second turn: a tool result is already in the conversation. Tests whether "
+            "the model handles a prior tool call/result correctly rather than looping."
+        ),
+        messages=[
+            {"role": "user", "content": "What is the weather in Oslo?"},
+        ],
+        tools=[_GET_WEATHER_TOOL],
+        multiturn_followup={
+            "role": "tool",
+            "content": '{"city": "Oslo", "condition": "cloudy", "temp_c": 8}',
+        },
+    ),
+]

caniollama-0.1.0/src/compat_check/suite/runner.py

@@ -0,0 +1,57 @@
+"""Executes the fixed case list against one model, producing CaseResults.
+
+Every case is isolated: one case erroring or timing out never aborts the run
+for the remaining cases.
+"""
+
+from __future__ import annotations
+
+import time
+
+from compat_check.models import CaseOutcome, CaseResult
+from compat_check.ollama_client import OllamaClient, OllamaError, OllamaRejected
+from compat_check.suite.cases import CASES, TestCase
+from compat_check.suite.scoring import score_response
+
+
+async def _run_case(client: OllamaClient, model: str, case: TestCase) -> CaseResult:
+    messages = list(case.messages)
+    started = time.monotonic()
+    try:
+        response = await client.chat(model=model, messages=messages, tools=case.tools)
+
+        if case.multiturn_followup:
+            # Feed the assistant's reply + a synthetic tool result back in, then
+            # score the model's *second* response -- this is what actually tests
+            # multi-turn tool-result handling, not just the first call.
+            messages.append(response.get("message", {}))
+            messages.append(case.multiturn_followup)
+            response = await client.chat(model=model, messages=messages, tools=case.tools)
+
+        outcome = score_response(case, response)
+        latency_ms = int((time.monotonic() - started) * 1000)
+        return CaseResult(case_id=case.case_id, outcome=outcome, latency_ms=latency_ms)
+
+    except OllamaRejected as exc:
+        latency_ms = int((time.monotonic() - started) * 1000)
+        return CaseResult(
+            case_id=case.case_id,
+            outcome=CaseOutcome.REJECTED_BY_BACKEND,
+            latency_ms=latency_ms,
+            detail=f"HTTP {exc.status_code}",
+        )
+    except OllamaError as exc:
+        latency_ms = int((time.monotonic() - started) * 1000)
+        return CaseResult(
+            case_id=case.case_id,
+            outcome=CaseOutcome.TIMEOUT_OR_ERROR,
+            latency_ms=latency_ms,
+            detail=str(exc)[:200],
+        )
+
+
+async def run_suite(client: OllamaClient, model: str) -> list[CaseResult]:
+    results = []
+    for case in CASES:
+        results.append(await _run_case(client, model, case))
+    return results

caniollama-0.1.0/src/compat_check/suite/scoring.py

@@ -0,0 +1,58 @@
+"""Classifies a raw /api/chat response (or a raised client exception) into a CaseOutcome.
+
+This is the piece most likely to need refinement as new models and Ollama
+versions are observed -- any change here is a SUITE_VERSION bump.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from compat_check.models import CaseOutcome
+from compat_check.suite.cases import TestCase
+
+
+def _content_looks_like_tool_call_json(content: str) -> bool:
+    """Heuristic for category 2: model knows the *shape* of a tool call but emits it
+    as JSON-ish text in `content` instead of the structured `tool_calls` field."""
+    stripped = content.strip()
+    if not stripped.startswith(("{", "[")):
+        return False
+    try:
+        parsed = json.loads(stripped)
+    except (json.JSONDecodeError, TypeError):
+        return False
+
+    candidates = parsed if isinstance(parsed, list) else [parsed]
+    for candidate in candidates:
+        if not isinstance(candidate, dict):
+            continue
+        keys = {k.lower() for k in candidate}
+        if {"name", "arguments"} <= keys or {"function", "arguments"} <= keys:
+            return True
+        if "tool_calls" in keys or "function_call" in keys:
+            return True
+    return False
+
+
+def score_response(case: TestCase, response: dict[str, Any]) -> CaseOutcome:
+    message = response.get("message", {})
+    tool_calls = message.get("tool_calls")
+    content = message.get("content", "") or ""
+
+    if case.expects_no_tool_call:
+        # A structured tool call here is arguably "too eager", but plumbing-wise the
+        # model still demonstrated structured calling -- so we still score it a pass
+        # for *capability*; over-eagerness is a quality concern, out of scope here.
+        if tool_calls:
+            return CaseOutcome.STRUCTURED_PASS
+        if _content_looks_like_tool_call_json(content):
+            return CaseOutcome.JSON_IN_CONTENT_FALLBACK
+        return CaseOutcome.STRUCTURED_PASS  # correctly answered without calling a tool
+
+    if tool_calls:
+        return CaseOutcome.STRUCTURED_PASS
+    if _content_looks_like_tool_call_json(content):
+        return CaseOutcome.JSON_IN_CONTENT_FALLBACK
+    return CaseOutcome.JSON_IN_CONTENT_FALLBACK if content else CaseOutcome.TIMEOUT_OR_ERROR

caniollama-0.1.0/src/compat_check/suite/version.py

@@ -0,0 +1,8 @@
+"""The test suite's own version, independent of the package's release version.
+
+Bump this any time a case prompt, a tool schema, or a scoring rule changes.
+Every submitted result is tagged with this so a 'pass' under an old, looser
+suite is never silently compared against a 'pass' under a newer, stricter one.
+"""
+
+SUITE_VERSION = "1.0.0"

caniollama-0.1.0/tests/conftest.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+import pytest
+
+from compat_check.models import RunReport
+
+
+@pytest.fixture
+def sample_run_report() -> RunReport:
+    from compat_check.models import CaseOutcome, CaseResult
+
+    return RunReport(
+        model_name="qwen2.5-coder:7b",
+        model_quantization="q4_K_M",
+        ollama_version="0.6.2",
+        suite_version="1.0.0",
+        os_name="linux",
+        results=[
+            CaseResult(case_id="single_tool_basic", outcome=CaseOutcome.STRUCTURED_PASS, latency_ms=120),
+            CaseResult(case_id="multi_tool_5x", outcome=CaseOutcome.STRUCTURED_PASS, latency_ms=180),
+            CaseResult(case_id="no_tools_needed_control", outcome=CaseOutcome.STRUCTURED_PASS, latency_ms=90),
+            CaseResult(case_id="multiturn_tool_result", outcome=CaseOutcome.JSON_IN_CONTENT_FALLBACK, latency_ms=200),
+        ],
+    )

caniollama-0.1.0/tests/test_cli.py

@@ -0,0 +1,16 @@
+from typer.testing import CliRunner
+
+from compat_check.cli import app
+
+runner = CliRunner()
+
+
+def test_version_command_runs():
+    result = runner.invoke(app, ["version"])
+    assert result.exit_code == 0
+    assert "caniollama" in result.stdout
+
+
+def test_check_fails_cleanly_when_ollama_unreachable():
+    result = runner.invoke(app, ["check", "--ollama-url", "http://127.0.0.1:1"])
+    assert result.exit_code == 1

caniollama-0.1.0/tests/test_scoring.py

@@ -0,0 +1,55 @@
+from compat_check.models import CaseOutcome
+from compat_check.suite.cases import CASES
+from compat_check.suite.scoring import score_response
+
+BASIC_CASE = next(c for c in CASES if c.case_id == "single_tool_basic")
+CONTROL_CASE = next(c for c in CASES if c.case_id == "no_tools_needed_control")
+
+
+def test_structured_tool_call_scores_pass():
+    response = {
+        "message": {
+            "role": "assistant",
+            "content": "",
+            "tool_calls": [
+                {"function": {"name": "get_weather", "arguments": {"city": "Tokyo"}}}
+            ],
+        }
+    }
+    assert score_response(BASIC_CASE, response) == CaseOutcome.STRUCTURED_PASS
+
+
+def test_json_in_content_scores_fallback():
+    response = {
+        "message": {
+            "role": "assistant",
+            "content": '{"name": "get_weather", "arguments": {"city": "Tokyo"}}',
+        }
+    }
+    assert score_response(BASIC_CASE, response) == CaseOutcome.JSON_IN_CONTENT_FALLBACK
+
+
+def test_plain_text_with_no_tool_call_scores_error():
+    response = {"message": {"role": "assistant", "content": ""}}
+    assert score_response(BASIC_CASE, response) == CaseOutcome.TIMEOUT_OR_ERROR
+
+
+def test_plain_prose_answer_scores_fallback_bucket():
+    response = {"message": {"role": "assistant", "content": "It's sunny in Tokyo today."}}
+    assert score_response(BASIC_CASE, response) == CaseOutcome.JSON_IN_CONTENT_FALLBACK
+
+
+def test_control_case_correctly_skips_tool_call():
+    response = {"message": {"role": "assistant", "content": "Bonjour !"}}
+    assert score_response(CONTROL_CASE, response) == CaseOutcome.STRUCTURED_PASS
+
+
+def test_control_case_still_passes_if_model_calls_tool_anyway():
+    response = {
+        "message": {
+            "role": "assistant",
+            "content": "",
+            "tool_calls": [{"function": {"name": "get_weather", "arguments": {"city": "?"}}}],
+        }
+    }
+    assert score_response(CONTROL_CASE, response) == CaseOutcome.STRUCTURED_PASS

caniollama-0.1.0/tests/test_submission.py

@@ -0,0 +1,67 @@
+"""Structural privacy guarantee tests: assert SubmissionPayload's schema cannot
+carry free text long enough to be a prompt or a model output, and that
+submission never touches case content."""
+
+import inspect
+
+import httpx
+import pytest
+
+from compat_check import submission
+from compat_check.models import SubmissionPayload
+from compat_check.suite.cases import CASES
+
+
+def test_submission_payload_fields_are_bounded():
+    """Every field must be a version tag, enum, coarse label, or timestamp --
+    never a free-text/content-shaped field."""
+    allowed_unbounded_str_fields = {"model_name", "model_quantization", "ollama_version"}
+    for name in SubmissionPayload.model_fields:
+        if name == "case_results":
+            # dict[str, CaseOutcome] -- values are constrained to the enum, not free text
+            continue
+        if name in allowed_unbounded_str_fields:
+            # These carry short identifiers (e.g. "qwen2.5-coder:7b"), not prompts/outputs
+            continue
+        # suite_version, os, timestamp: also bounded/short by construction
+        assert name in {"suite_version", "os", "timestamp"}, (
+            f"Unexpected field '{name}' on SubmissionPayload -- review for privacy leakage"
+        )
+
+
+def test_submission_module_never_imports_case_prompts():
+    """submission.py's *code* (not comments/docstrings) should never touch case
+    message/prompt content."""
+    source = inspect.getsource(submission)
+    code_lines = [
+        line for line in source.splitlines()
+        if line.strip() and not line.strip().startswith(("#", '"""', "'''"))
+    ]
+    code_only = "\n".join(code_lines)
+    forbidden_tokens = ["case.messages", "case.tools", 'message.get("content']
+    for token in forbidden_tokens:
+        assert token not in code_only, (
+            f"submission.py references '{token}' in executable code -- privacy violation risk"
+        )
+
+
+def test_to_submission_payload_only_carries_outcomes(sample_run_report):
+    payload = sample_run_report.to_submission_payload()
+    for case_id, outcome in payload.case_results.items():
+        assert case_id in {c.case_id for c in CASES}
+        assert outcome in {
+            "structured_pass",
+            "json_fallback",
+            "rejected",
+            "error",
+        }
+
+
+@pytest.mark.asyncio
+async def test_submit_result_returns_false_on_network_error(monkeypatch, sample_run_report):
+    async def _boom(*args, **kwargs):
+        raise httpx.ConnectError("no route to host")
+
+    monkeypatch.setattr(httpx.AsyncClient, "post", _boom)
+    ok = await submission.submit_result(sample_run_report, base_url="http://unreachable.invalid")
+    assert ok is False