breed-registry 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- breed_registry-1.0.0/LICENSE +21 -0
- breed_registry-1.0.0/PKG-INFO +120 -0
- breed_registry-1.0.0/README.md +97 -0
- breed_registry-1.0.0/pyproject.toml +42 -0
- breed_registry-1.0.0/setup.cfg +4 -0
- breed_registry-1.0.0/src/breed_registry/__init__.py +45 -0
- breed_registry-1.0.0/src/breed_registry/matcher.py +233 -0
- breed_registry-1.0.0/src/breed_registry/models.py +94 -0
- breed_registry-1.0.0/src/breed_registry.egg-info/PKG-INFO +120 -0
- breed_registry-1.0.0/src/breed_registry.egg-info/SOURCES.txt +11 -0
- breed_registry-1.0.0/src/breed_registry.egg-info/dependency_links.txt +1 -0
- breed_registry-1.0.0/src/breed_registry.egg-info/top_level.txt +1 -0
- breed_registry-1.0.0/tests/test_matcher.py +212 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 SuperInstance
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: breed-registry
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: The Breed Registry — model selection as breeding selection, from Working Animal Architecture
|
|
5
|
+
Author: SuperInstance
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/SuperInstance/breed-registry
|
|
8
|
+
Project-URL: Repository, https://github.com/SuperInstance/breed-registry
|
|
9
|
+
Keywords: llm,model-selection,ai,working-animal-architecture
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Requires-Python: >=3.9
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# 🐕 The Breed Registry
|
|
25
|
+
|
|
26
|
+
> **Model selection as breeding selection.** In the working dog paradigm, choosing the right breed is the most important decision you'll make. This registry provides structured guidance on which base model to use for which task.
|
|
27
|
+
|
|
28
|
+
## Why a Breed Registry?
|
|
29
|
+
|
|
30
|
+
A working dog isn't a pet — it's bred for a purpose. Border collies herd. Labs retrieve. German shepherds protect. Each breed has instincts, temperaments, and physical traits honed over generations of selective breeding.
|
|
31
|
+
|
|
32
|
+
Foundation models are the same. GPT-4 is a generalist — like a German shepherd that can do everything. Llama-3 is an open-lineage working dog you can train yourself. Mistral is a lean, efficient breed from European lines.
|
|
33
|
+
|
|
34
|
+
**Choosing the wrong breed for the job wastes resources and produces poor results.** This registry exists to prevent that.
|
|
35
|
+
|
|
36
|
+
---
|
|
37
|
+
|
|
38
|
+
## What's Inside
|
|
39
|
+
|
|
40
|
+
### `registry/` — Breed Assessments
|
|
41
|
+
|
|
42
|
+
Each model is assessed like a dog breed at a confirmation show, but for **working aptitude** instead of conformation:
|
|
43
|
+
|
|
44
|
+
| Field | Dog Analogy | Model Meaning |
|
|
45
|
+
|---|---|---|
|
|
46
|
+
| `lineage` | Pedigree / bloodline | Model family and training lineage |
|
|
47
|
+
| `breed_group` | AKC group (Herding, Sporting, etc.) | Task category (General Purpose, Working, etc.) |
|
|
48
|
+
| `temperament` | Breed temperament traits | Behavioral characteristics (careful, verbose, etc.) |
|
|
49
|
+
| `working_aptitude` | Instinct tests & working trials | Task-specific capability scores (0-10) |
|
|
50
|
+
| `cost_profile` | Feed & maintenance cost | API cost per token |
|
|
51
|
+
| `speed_profile` | Speed/agility rating | Inference latency |
|
|
52
|
+
| `trainability` | How well it takes to training | Fine-tune response quality |
|
|
53
|
+
| `recommended_for` | Best-suited jobs | Tasks where this model excels |
|
|
54
|
+
| `not_recommended_for` | Jobs it's wrong for | Tasks where this model struggles |
|
|
55
|
+
| `fence_compatibility` | How well it respects boundaries | Conservation bytecode compliance |
|
|
56
|
+
|
|
57
|
+
### `src/breed_registry/` — Python API
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
from breed_registry import select_breed, compare_breeds, assess_aptitude
|
|
61
|
+
|
|
62
|
+
# Find the best breed for a task
|
|
63
|
+
recommendations = select_breed("code_generation")
|
|
64
|
+
|
|
65
|
+
# Compare two breeds head-to-head
|
|
66
|
+
report = compare_breeds("gpt-4", "llama-3")
|
|
67
|
+
|
|
68
|
+
# Check a specific model's aptitude for a task
|
|
69
|
+
score = assess_aptitude("mistral", "analysis")
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
---
|
|
73
|
+
|
|
74
|
+
## Registered Breeds
|
|
75
|
+
|
|
76
|
+
| Breed | Group | Cost | Speed | Best For |
|
|
77
|
+
|---|---|---|---|---|
|
|
78
|
+
| [GPT-4](registry/gpt-4.json) | General Purpose | High | Moderate | Complex reasoning, code generation |
|
|
79
|
+
| [Claude-3](registry/claude-3.json | General Purpose | High | Moderate | Analysis, careful reasoning |
|
|
80
|
+
| [Llama-3](registry/llama-3.json) | Open Lineage | Free | Fast | Local deployment, fine-tuning |
|
|
81
|
+
| [GLM](registry/glm.json) | General Purpose | Low | Fast | Multilingual, efficient inference |
|
|
82
|
+
| [Mistral](registry/mistral.json) | Working | Low | Fast | Efficient European-lineage tasks |
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
|
|
86
|
+
## Selection Philosophy
|
|
87
|
+
|
|
88
|
+
### 1. Match the Breed to the Job
|
|
89
|
+
Don't use a high-cost generalist for simple classification. Don't use a lightweight model for complex multi-step reasoning. Match instincts to tasks.
|
|
90
|
+
|
|
91
|
+
### 2. Consider the Lineage
|
|
92
|
+
Model families carry traits across generations. Fine-tunes inherit strengths and weaknesses. Know the pedigree before you commit.
|
|
93
|
+
|
|
94
|
+
### 3. Respect the Fence
|
|
95
|
+
Conservation compliance — how well a model respects guardrails and bytecode fences — is non-negotiable for production work. A model that won't respect boundaries is a liability.
|
|
96
|
+
|
|
97
|
+
### 4. Trainability Matters
|
|
98
|
+
Some breeds take to fine-tuning naturally. Others resist. If you need to specialize, choose a breed known for trainability.
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## Adding a Breed
|
|
103
|
+
|
|
104
|
+
To register a new model:
|
|
105
|
+
|
|
106
|
+
1. Create `registry/<model-name>.json` following the assessment schema
|
|
107
|
+
2. Add an entry to `registry/index.json`
|
|
108
|
+
3. Submit a pull request
|
|
109
|
+
|
|
110
|
+
Assess honestly. Overrating a model's working aptitude helps no one — least of all the teams who deploy it.
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
## License
|
|
115
|
+
|
|
116
|
+
MIT
|
|
117
|
+
|
|
118
|
+
---
|
|
119
|
+
|
|
120
|
+
*Part of the [Working Animal Architecture](https://github.com/SuperInstance) — foundation models as working animals, not pets.*
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# 🐕 The Breed Registry
|
|
2
|
+
|
|
3
|
+
> **Model selection as breeding selection.** In the working dog paradigm, choosing the right breed is the most important decision you'll make. This registry provides structured guidance on which base model to use for which task.
|
|
4
|
+
|
|
5
|
+
## Why a Breed Registry?
|
|
6
|
+
|
|
7
|
+
A working dog isn't a pet — it's bred for a purpose. Border collies herd. Labs retrieve. German shepherds protect. Each breed has instincts, temperaments, and physical traits honed over generations of selective breeding.
|
|
8
|
+
|
|
9
|
+
Foundation models are the same. GPT-4 is a generalist — like a German shepherd that can do everything. Llama-3 is an open-lineage working dog you can train yourself. Mistral is a lean, efficient breed from European lines.
|
|
10
|
+
|
|
11
|
+
**Choosing the wrong breed for the job wastes resources and produces poor results.** This registry exists to prevent that.
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## What's Inside
|
|
16
|
+
|
|
17
|
+
### `registry/` — Breed Assessments
|
|
18
|
+
|
|
19
|
+
Each model is assessed like a dog breed at a confirmation show, but for **working aptitude** instead of conformation:
|
|
20
|
+
|
|
21
|
+
| Field | Dog Analogy | Model Meaning |
|
|
22
|
+
|---|---|---|
|
|
23
|
+
| `lineage` | Pedigree / bloodline | Model family and training lineage |
|
|
24
|
+
| `breed_group` | AKC group (Herding, Sporting, etc.) | Task category (General Purpose, Working, etc.) |
|
|
25
|
+
| `temperament` | Breed temperament traits | Behavioral characteristics (careful, verbose, etc.) |
|
|
26
|
+
| `working_aptitude` | Instinct tests & working trials | Task-specific capability scores (0-10) |
|
|
27
|
+
| `cost_profile` | Feed & maintenance cost | API cost per token |
|
|
28
|
+
| `speed_profile` | Speed/agility rating | Inference latency |
|
|
29
|
+
| `trainability` | How well it takes to training | Fine-tune response quality |
|
|
30
|
+
| `recommended_for` | Best-suited jobs | Tasks where this model excels |
|
|
31
|
+
| `not_recommended_for` | Jobs it's wrong for | Tasks where this model struggles |
|
|
32
|
+
| `fence_compatibility` | How well it respects boundaries | Conservation bytecode compliance |
|
|
33
|
+
|
|
34
|
+
### `src/breed_registry/` — Python API
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
from breed_registry import select_breed, compare_breeds, assess_aptitude
|
|
38
|
+
|
|
39
|
+
# Find the best breed for a task
|
|
40
|
+
recommendations = select_breed("code_generation")
|
|
41
|
+
|
|
42
|
+
# Compare two breeds head-to-head
|
|
43
|
+
report = compare_breeds("gpt-4", "llama-3")
|
|
44
|
+
|
|
45
|
+
# Check a specific model's aptitude for a task
|
|
46
|
+
score = assess_aptitude("mistral", "analysis")
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
---
|
|
50
|
+
|
|
51
|
+
## Registered Breeds
|
|
52
|
+
|
|
53
|
+
| Breed | Group | Cost | Speed | Best For |
|
|
54
|
+
|---|---|---|---|---|
|
|
55
|
+
| [GPT-4](registry/gpt-4.json) | General Purpose | High | Moderate | Complex reasoning, code generation |
|
|
56
|
+
| [Claude-3](registry/claude-3.json | General Purpose | High | Moderate | Analysis, careful reasoning |
|
|
57
|
+
| [Llama-3](registry/llama-3.json) | Open Lineage | Free | Fast | Local deployment, fine-tuning |
|
|
58
|
+
| [GLM](registry/glm.json) | General Purpose | Low | Fast | Multilingual, efficient inference |
|
|
59
|
+
| [Mistral](registry/mistral.json) | Working | Low | Fast | Efficient European-lineage tasks |
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## Selection Philosophy
|
|
64
|
+
|
|
65
|
+
### 1. Match the Breed to the Job
|
|
66
|
+
Don't use a high-cost generalist for simple classification. Don't use a lightweight model for complex multi-step reasoning. Match instincts to tasks.
|
|
67
|
+
|
|
68
|
+
### 2. Consider the Lineage
|
|
69
|
+
Model families carry traits across generations. Fine-tunes inherit strengths and weaknesses. Know the pedigree before you commit.
|
|
70
|
+
|
|
71
|
+
### 3. Respect the Fence
|
|
72
|
+
Conservation compliance — how well a model respects guardrails and bytecode fences — is non-negotiable for production work. A model that won't respect boundaries is a liability.
|
|
73
|
+
|
|
74
|
+
### 4. Trainability Matters
|
|
75
|
+
Some breeds take to fine-tuning naturally. Others resist. If you need to specialize, choose a breed known for trainability.
|
|
76
|
+
|
|
77
|
+
---
|
|
78
|
+
|
|
79
|
+
## Adding a Breed
|
|
80
|
+
|
|
81
|
+
To register a new model:
|
|
82
|
+
|
|
83
|
+
1. Create `registry/<model-name>.json` following the assessment schema
|
|
84
|
+
2. Add an entry to `registry/index.json`
|
|
85
|
+
3. Submit a pull request
|
|
86
|
+
|
|
87
|
+
Assess honestly. Overrating a model's working aptitude helps no one — least of all the teams who deploy it.
|
|
88
|
+
|
|
89
|
+
---
|
|
90
|
+
|
|
91
|
+
## License
|
|
92
|
+
|
|
93
|
+
MIT
|
|
94
|
+
|
|
95
|
+
---
|
|
96
|
+
|
|
97
|
+
*Part of the [Working Animal Architecture](https://github.com/SuperInstance) — foundation models as working animals, not pets.*
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "breed-registry"
|
|
7
|
+
version = "1.0.0"
|
|
8
|
+
description = "The Breed Registry — model selection as breeding selection, from Working Animal Architecture"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = {text = "MIT"}
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
authors = [
|
|
13
|
+
{name = "SuperInstance"},
|
|
14
|
+
]
|
|
15
|
+
keywords = ["llm", "model-selection", "ai", "working-animal-architecture"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.9",
|
|
22
|
+
"Programming Language :: Python :: 3.10",
|
|
23
|
+
"Programming Language :: Python :: 3.11",
|
|
24
|
+
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Homepage = "https://github.com/SuperInstance/breed-registry"
|
|
30
|
+
Repository = "https://github.com/SuperInstance/breed-registry"
|
|
31
|
+
|
|
32
|
+
[tool.setuptools.packages.find]
|
|
33
|
+
where = ["src"]
|
|
34
|
+
|
|
35
|
+
[tool.pytest.ini_options]
|
|
36
|
+
testpaths = ["tests"]
|
|
37
|
+
pythonpath = ["src"]
|
|
38
|
+
|
|
39
|
+
[dependency-groups]
|
|
40
|
+
dev = [
|
|
41
|
+
"pytest>=7.0",
|
|
42
|
+
]
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The Breed Registry — model selection as breeding selection.
|
|
3
|
+
|
|
4
|
+
In the working dog paradigm, choosing the right breed is the most important
|
|
5
|
+
decision. This library provides structured guidance on which base model to
|
|
6
|
+
use for which task.
|
|
7
|
+
|
|
8
|
+
Usage:
|
|
9
|
+
from breed_registry import select_breed, compare_breeds, assess_aptitude
|
|
10
|
+
|
|
11
|
+
# Find the best breed for a task
|
|
12
|
+
recommendations = select_breed("code_generation")
|
|
13
|
+
|
|
14
|
+
# Compare two breeds head-to-head
|
|
15
|
+
report = compare_breeds("gpt-4", "llama-3")
|
|
16
|
+
|
|
17
|
+
# Check a specific model's aptitude for a task
|
|
18
|
+
score = assess_aptitude("mistral", "analysis")
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from breed_registry.matcher import (
|
|
22
|
+
select_breed,
|
|
23
|
+
compare_breeds,
|
|
24
|
+
assess_aptitude,
|
|
25
|
+
list_breeds,
|
|
26
|
+
get_breed,
|
|
27
|
+
)
|
|
28
|
+
from breed_registry.models import (
|
|
29
|
+
ModelAssessment,
|
|
30
|
+
ComparisonReport,
|
|
31
|
+
AptitudeScore,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
__version__ = "1.0.0"
|
|
35
|
+
|
|
36
|
+
__all__ = [
|
|
37
|
+
"select_breed",
|
|
38
|
+
"compare_breeds",
|
|
39
|
+
"assess_aptitude",
|
|
40
|
+
"list_breeds",
|
|
41
|
+
"get_breed",
|
|
42
|
+
"ModelAssessment",
|
|
43
|
+
"ComparisonReport",
|
|
44
|
+
"AptitudeScore",
|
|
45
|
+
]
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Task-to-model matcher — the core selection engine.
|
|
3
|
+
|
|
4
|
+
Given a task (e.g. "code_generation"), recommends the best model breeds
|
|
5
|
+
based on their working aptitude scores.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Dict, List, Optional, Tuple
|
|
13
|
+
|
|
14
|
+
from breed_registry.models import (
|
|
15
|
+
AptitudeScore,
|
|
16
|
+
ComparisonReport,
|
|
17
|
+
ModelAssessment,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
# ---------------------------------------------------------------------------
|
|
21
|
+
# Registry loading
|
|
22
|
+
# ---------------------------------------------------------------------------
|
|
23
|
+
|
|
24
|
+
_REGISTRY_DIR = Path(__file__).resolve().parent.parent.parent / "registry"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _load_registry(registry_dir: Optional[Path] = None) -> Dict[str, ModelAssessment]:
|
|
28
|
+
"""Load all breed assessments from the registry directory."""
|
|
29
|
+
reg_dir = registry_dir or _REGISTRY_DIR
|
|
30
|
+
assessments: Dict[str, ModelAssessment] = {}
|
|
31
|
+
|
|
32
|
+
index_path = reg_dir / "index.json"
|
|
33
|
+
if not index_path.exists():
|
|
34
|
+
raise FileNotFoundError(f"Registry index not found at {index_path}")
|
|
35
|
+
|
|
36
|
+
with open(index_path) as f:
|
|
37
|
+
index = json.load(f)
|
|
38
|
+
|
|
39
|
+
for key, info in index.get("breeds", {}).items():
|
|
40
|
+
breed_file = reg_dir / info["file"]
|
|
41
|
+
if breed_file.exists():
|
|
42
|
+
with open(breed_file) as f:
|
|
43
|
+
data = json.load(f)
|
|
44
|
+
assessments[key] = ModelAssessment.from_dict(data)
|
|
45
|
+
|
|
46
|
+
return assessments
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# Lazy-loaded global registry
|
|
50
|
+
_REGISTRY_CACHE: Optional[Dict[str, ModelAssessment]] = None
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _get_registry() -> Dict[str, ModelAssessment]:
|
|
54
|
+
global _REGISTRY_CACHE
|
|
55
|
+
if _REGISTRY_CACHE is None:
|
|
56
|
+
_REGISTRY_CACHE = _load_registry()
|
|
57
|
+
return _REGISTRY_CACHE
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _score_to_rating(score: int) -> str:
|
|
61
|
+
"""Convert a numeric score (0-10) to a human-readable rating."""
|
|
62
|
+
if score >= 9:
|
|
63
|
+
return "excellent"
|
|
64
|
+
elif score >= 7:
|
|
65
|
+
return "good"
|
|
66
|
+
elif score >= 5:
|
|
67
|
+
return "fair"
|
|
68
|
+
elif score >= 3:
|
|
69
|
+
return "poor"
|
|
70
|
+
else:
|
|
71
|
+
return "unsuitable"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
# ---------------------------------------------------------------------------
|
|
75
|
+
# Public API
|
|
76
|
+
# ---------------------------------------------------------------------------
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def list_breeds() -> List[str]:
|
|
80
|
+
"""List all registered breed names."""
|
|
81
|
+
return sorted(_get_registry().keys())
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def get_breed(name: str) -> ModelAssessment:
|
|
85
|
+
"""Get the full assessment for a specific breed.
|
|
86
|
+
|
|
87
|
+
Raises:
|
|
88
|
+
KeyError: if the breed is not registered.
|
|
89
|
+
"""
|
|
90
|
+
registry = _get_registry()
|
|
91
|
+
if name not in registry:
|
|
92
|
+
raise KeyError(
|
|
93
|
+
f"Breed '{name}' not registered. Available: {', '.join(sorted(registry.keys()))}"
|
|
94
|
+
)
|
|
95
|
+
return registry[name]
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def select_breed(
|
|
99
|
+
task: str,
|
|
100
|
+
top_k: int = 3,
|
|
101
|
+
max_cost: Optional[str] = None,
|
|
102
|
+
) -> List[ModelAssessment]:
|
|
103
|
+
"""Recommend models for a given task, sorted by aptitude.
|
|
104
|
+
|
|
105
|
+
Args:
|
|
106
|
+
task: The task category (e.g. "code_generation", "analysis").
|
|
107
|
+
top_k: Maximum number of recommendations to return.
|
|
108
|
+
max_cost: Optional cost ceiling — "low" excludes moderate/high,
|
|
109
|
+
"free" excludes everything but free models.
|
|
110
|
+
|
|
111
|
+
Returns:
|
|
112
|
+
List of ModelAssessment objects, best match first.
|
|
113
|
+
"""
|
|
114
|
+
registry = _get_registry()
|
|
115
|
+
cost_order = {"free": 0, "low": 1, "moderate": 2, "high": 3}
|
|
116
|
+
cost_ceiling = cost_order.get(max_cost, 3) if max_cost else 3
|
|
117
|
+
|
|
118
|
+
scored: List[Tuple[int, ModelAssessment]] = []
|
|
119
|
+
for assessment in registry.values():
|
|
120
|
+
# Filter by cost ceiling
|
|
121
|
+
model_cost = cost_order.get(assessment.cost_profile, 3)
|
|
122
|
+
if model_cost > cost_ceiling:
|
|
123
|
+
continue
|
|
124
|
+
|
|
125
|
+
score = assessment.aptitude_for(task)
|
|
126
|
+
if score > 0:
|
|
127
|
+
scored.append((score, assessment))
|
|
128
|
+
|
|
129
|
+
scored.sort(key=lambda x: x[0], reverse=True)
|
|
130
|
+
|
|
131
|
+
return [assessment for _, assessment in scored[:top_k]]
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def compare_breeds(model_a: str, model_b: str) -> ComparisonReport:
|
|
135
|
+
"""Compare two breeds head-to-head across all working aptitude dimensions.
|
|
136
|
+
|
|
137
|
+
Raises:
|
|
138
|
+
KeyError: if either model is not registered.
|
|
139
|
+
"""
|
|
140
|
+
breed_a = get_breed(model_a)
|
|
141
|
+
breed_b = get_breed(model_b)
|
|
142
|
+
|
|
143
|
+
# Build task-by-task comparison
|
|
144
|
+
all_tasks = sorted(
|
|
145
|
+
set(breed_a.working_aptitude.keys()) | set(breed_b.working_aptitude.keys())
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
aptitude_comparison: Dict[str, Dict[str, int]] = {}
|
|
149
|
+
advantages_a: List[str] = []
|
|
150
|
+
advantages_b: List[str] = []
|
|
151
|
+
total_a = 0
|
|
152
|
+
total_b = 0
|
|
153
|
+
|
|
154
|
+
for task in all_tasks:
|
|
155
|
+
score_a = breed_a.aptitude_for(task)
|
|
156
|
+
score_b = breed_b.aptitude_for(task)
|
|
157
|
+
aptitude_comparison[task] = {model_a: score_a, model_b: score_b}
|
|
158
|
+
total_a += score_a
|
|
159
|
+
total_b += score_b
|
|
160
|
+
|
|
161
|
+
if score_a > score_b:
|
|
162
|
+
advantages_a.append(task)
|
|
163
|
+
elif score_b > score_a:
|
|
164
|
+
advantages_b.append(task)
|
|
165
|
+
|
|
166
|
+
margin = abs(total_a - total_b)
|
|
167
|
+
if total_a > total_b:
|
|
168
|
+
winner = model_a
|
|
169
|
+
elif total_b > total_a:
|
|
170
|
+
winner = model_b
|
|
171
|
+
else:
|
|
172
|
+
winner = "tie"
|
|
173
|
+
|
|
174
|
+
cost_order = {"free": 0, "low": 1, "moderate": 2, "high": 3}
|
|
175
|
+
cost_a = cost_order.get(breed_a.cost_profile, 3)
|
|
176
|
+
cost_b = cost_order.get(breed_b.cost_profile, 3)
|
|
177
|
+
|
|
178
|
+
if cost_a < cost_b:
|
|
179
|
+
cost_notes = f"{model_a} is cheaper ({breed_a.cost_profile} vs {breed_b.cost_profile})"
|
|
180
|
+
elif cost_b < cost_a:
|
|
181
|
+
cost_notes = f"{model_b} is cheaper ({breed_b.cost_profile} vs {breed_a.cost_profile})"
|
|
182
|
+
else:
|
|
183
|
+
cost_notes = f"Both are {breed_a.cost_profile} cost"
|
|
184
|
+
|
|
185
|
+
speed_order = {"fast": 0, "moderate": 1, "slow": 2}
|
|
186
|
+
speed_a = speed_order.get(breed_a.speed_profile, 2)
|
|
187
|
+
speed_b = speed_order.get(breed_b.speed_profile, 2)
|
|
188
|
+
|
|
189
|
+
if speed_a < speed_b:
|
|
190
|
+
speed_notes = f"{model_a} is faster ({breed_a.speed_profile} vs {breed_b.speed_profile})"
|
|
191
|
+
elif speed_b < speed_a:
|
|
192
|
+
speed_notes = f"{model_b} is faster ({breed_b.speed_profile} vs {breed_a.speed_profile})"
|
|
193
|
+
else:
|
|
194
|
+
speed_notes = f"Both are {breed_a.speed_profile} speed"
|
|
195
|
+
|
|
196
|
+
return ComparisonReport(
|
|
197
|
+
model_a=model_a,
|
|
198
|
+
model_b=model_b,
|
|
199
|
+
aptitude_comparison=aptitude_comparison,
|
|
200
|
+
winner=winner,
|
|
201
|
+
margin=margin,
|
|
202
|
+
advantages_a=advantages_a,
|
|
203
|
+
advantages_b=advantages_b,
|
|
204
|
+
cost_notes=cost_notes,
|
|
205
|
+
speed_notes=speed_notes,
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def assess_aptitude(model: str, task: str) -> AptitudeScore:
|
|
210
|
+
"""Assess a specific model's aptitude for a specific task.
|
|
211
|
+
|
|
212
|
+
Raises:
|
|
213
|
+
KeyError: if the model is not registered.
|
|
214
|
+
"""
|
|
215
|
+
breed = get_breed(model)
|
|
216
|
+
score = breed.aptitude_for(task)
|
|
217
|
+
|
|
218
|
+
# Calculate percentile
|
|
219
|
+
registry = _get_registry()
|
|
220
|
+
all_scores = [a.aptitude_for(task) for a in registry.values() if a.aptitude_for(task) > 0]
|
|
221
|
+
if all_scores:
|
|
222
|
+
below = sum(1 for s in all_scores if s < score)
|
|
223
|
+
percentile = round((below / len(all_scores)) * 100, 1)
|
|
224
|
+
else:
|
|
225
|
+
percentile = None
|
|
226
|
+
|
|
227
|
+
return AptitudeScore(
|
|
228
|
+
model=model,
|
|
229
|
+
task=task,
|
|
230
|
+
score=score,
|
|
231
|
+
rating=_score_to_rating(score),
|
|
232
|
+
percentile=percentile,
|
|
233
|
+
)
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Data models for the Breed Registry.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import Dict, List, Optional
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass
|
|
12
|
+
class AptitudeScore:
|
|
13
|
+
"""A model's aptitude score for a specific task."""
|
|
14
|
+
|
|
15
|
+
model: str
|
|
16
|
+
task: str
|
|
17
|
+
score: int # 0-10
|
|
18
|
+
rating: str # human-readable rating
|
|
19
|
+
percentile: Optional[float] = None # percentile among all assessed models
|
|
20
|
+
|
|
21
|
+
def __repr__(self) -> str:
|
|
22
|
+
return f"AptitudeScore(model={self.model!r}, task={self.task!r}, score={self.score}/10, rating={self.rating!r})"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class ComparisonReport:
|
|
27
|
+
"""Head-to-head comparison between two models."""
|
|
28
|
+
|
|
29
|
+
model_a: str
|
|
30
|
+
model_b: str
|
|
31
|
+
aptitude_comparison: Dict[str, Dict[str, int]] # task -> {model_a: score, model_b: score}
|
|
32
|
+
winner: str # overall winner
|
|
33
|
+
margin: int # total score difference
|
|
34
|
+
advantages_a: List[str] # tasks where model_a wins
|
|
35
|
+
advantages_b: List[str] # tasks where model_b wins
|
|
36
|
+
cost_notes: str
|
|
37
|
+
speed_notes: str
|
|
38
|
+
|
|
39
|
+
def summary(self) -> str:
|
|
40
|
+
lines = [
|
|
41
|
+
f"Comparison: {self.model_a} vs {self.model_b}",
|
|
42
|
+
f"Overall winner: {self.winner} (margin: {self.margin} points)",
|
|
43
|
+
"",
|
|
44
|
+
f"Advantages {self.model_a}: {', '.join(self.advantages_a) or 'none'}",
|
|
45
|
+
f"Advantages {self.model_b}: {', '.join(self.advantages_b) or 'none'}",
|
|
46
|
+
"",
|
|
47
|
+
f"Cost: {self.cost_notes}",
|
|
48
|
+
f"Speed: {self.speed_notes}",
|
|
49
|
+
]
|
|
50
|
+
return "\n".join(lines)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass
|
|
54
|
+
class ModelAssessment:
|
|
55
|
+
"""Full breed assessment for a model."""
|
|
56
|
+
|
|
57
|
+
name: str
|
|
58
|
+
lineage: str
|
|
59
|
+
breed_group: str
|
|
60
|
+
temperament: List[str]
|
|
61
|
+
working_aptitude: Dict[str, int]
|
|
62
|
+
cost_profile: str
|
|
63
|
+
speed_profile: str
|
|
64
|
+
trainability: str
|
|
65
|
+
recommended_for: List[str]
|
|
66
|
+
not_recommended_for: List[str]
|
|
67
|
+
fence_compatibility: str
|
|
68
|
+
notes: Optional[str] = None
|
|
69
|
+
|
|
70
|
+
@classmethod
|
|
71
|
+
def from_dict(cls, data: dict) -> "ModelAssessment":
|
|
72
|
+
return cls(
|
|
73
|
+
name=data["name"],
|
|
74
|
+
lineage=data["lineage"],
|
|
75
|
+
breed_group=data["breed_group"],
|
|
76
|
+
temperament=data["temperament"],
|
|
77
|
+
working_aptitude=data["working_aptitude"],
|
|
78
|
+
cost_profile=data["cost_profile"],
|
|
79
|
+
speed_profile=data["speed_profile"],
|
|
80
|
+
trainability=data["trainability"],
|
|
81
|
+
recommended_for=data["recommended_for"],
|
|
82
|
+
not_recommended_for=data["not_recommended_for"],
|
|
83
|
+
fence_compatibility=data["fence_compatibility"],
|
|
84
|
+
notes=data.get("notes"),
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
def aptitude_for(self, task: str) -> int:
|
|
88
|
+
"""Return the aptitude score (0-10) for a given task."""
|
|
89
|
+
return self.working_aptitude.get(task, 0)
|
|
90
|
+
|
|
91
|
+
def overall_score(self) -> float:
|
|
92
|
+
"""Average aptitude across all assessed tasks."""
|
|
93
|
+
scores = list(self.working_aptitude.values())
|
|
94
|
+
return sum(scores) / len(scores) if scores else 0.0
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: breed-registry
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: The Breed Registry — model selection as breeding selection, from Working Animal Architecture
|
|
5
|
+
Author: SuperInstance
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/SuperInstance/breed-registry
|
|
8
|
+
Project-URL: Repository, https://github.com/SuperInstance/breed-registry
|
|
9
|
+
Keywords: llm,model-selection,ai,working-animal-architecture
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Requires-Python: >=3.9
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# 🐕 The Breed Registry
|
|
25
|
+
|
|
26
|
+
> **Model selection as breeding selection.** In the working dog paradigm, choosing the right breed is the most important decision you'll make. This registry provides structured guidance on which base model to use for which task.
|
|
27
|
+
|
|
28
|
+
## Why a Breed Registry?
|
|
29
|
+
|
|
30
|
+
A working dog isn't a pet — it's bred for a purpose. Border collies herd. Labs retrieve. German shepherds protect. Each breed has instincts, temperaments, and physical traits honed over generations of selective breeding.
|
|
31
|
+
|
|
32
|
+
Foundation models are the same. GPT-4 is a generalist — like a German shepherd that can do everything. Llama-3 is an open-lineage working dog you can train yourself. Mistral is a lean, efficient breed from European lines.
|
|
33
|
+
|
|
34
|
+
**Choosing the wrong breed for the job wastes resources and produces poor results.** This registry exists to prevent that.
|
|
35
|
+
|
|
36
|
+
---
|
|
37
|
+
|
|
38
|
+
## What's Inside
|
|
39
|
+
|
|
40
|
+
### `registry/` — Breed Assessments
|
|
41
|
+
|
|
42
|
+
Each model is assessed like a dog breed at a confirmation show, but for **working aptitude** instead of conformation:
|
|
43
|
+
|
|
44
|
+
| Field | Dog Analogy | Model Meaning |
|
|
45
|
+
|---|---|---|
|
|
46
|
+
| `lineage` | Pedigree / bloodline | Model family and training lineage |
|
|
47
|
+
| `breed_group` | AKC group (Herding, Sporting, etc.) | Task category (General Purpose, Working, etc.) |
|
|
48
|
+
| `temperament` | Breed temperament traits | Behavioral characteristics (careful, verbose, etc.) |
|
|
49
|
+
| `working_aptitude` | Instinct tests & working trials | Task-specific capability scores (0-10) |
|
|
50
|
+
| `cost_profile` | Feed & maintenance cost | API cost per token |
|
|
51
|
+
| `speed_profile` | Speed/agility rating | Inference latency |
|
|
52
|
+
| `trainability` | How well it takes to training | Fine-tune response quality |
|
|
53
|
+
| `recommended_for` | Best-suited jobs | Tasks where this model excels |
|
|
54
|
+
| `not_recommended_for` | Jobs it's wrong for | Tasks where this model struggles |
|
|
55
|
+
| `fence_compatibility` | How well it respects boundaries | Conservation bytecode compliance |
|
|
56
|
+
|
|
57
|
+
### `src/breed_registry/` — Python API
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
from breed_registry import select_breed, compare_breeds, assess_aptitude
|
|
61
|
+
|
|
62
|
+
# Find the best breed for a task
|
|
63
|
+
recommendations = select_breed("code_generation")
|
|
64
|
+
|
|
65
|
+
# Compare two breeds head-to-head
|
|
66
|
+
report = compare_breeds("gpt-4", "llama-3")
|
|
67
|
+
|
|
68
|
+
# Check a specific model's aptitude for a task
|
|
69
|
+
score = assess_aptitude("mistral", "analysis")
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
---
|
|
73
|
+
|
|
74
|
+
## Registered Breeds
|
|
75
|
+
|
|
76
|
+
| Breed | Group | Cost | Speed | Best For |
|
|
77
|
+
|---|---|---|---|---|
|
|
78
|
+
| [GPT-4](registry/gpt-4.json) | General Purpose | High | Moderate | Complex reasoning, code generation |
|
|
79
|
+
| [Claude-3](registry/claude-3.json | General Purpose | High | Moderate | Analysis, careful reasoning |
|
|
80
|
+
| [Llama-3](registry/llama-3.json) | Open Lineage | Free | Fast | Local deployment, fine-tuning |
|
|
81
|
+
| [GLM](registry/glm.json) | General Purpose | Low | Fast | Multilingual, efficient inference |
|
|
82
|
+
| [Mistral](registry/mistral.json) | Working | Low | Fast | Efficient European-lineage tasks |
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
|
|
86
|
+
## Selection Philosophy
|
|
87
|
+
|
|
88
|
+
### 1. Match the Breed to the Job
|
|
89
|
+
Don't use a high-cost generalist for simple classification. Don't use a lightweight model for complex multi-step reasoning. Match instincts to tasks.
|
|
90
|
+
|
|
91
|
+
### 2. Consider the Lineage
|
|
92
|
+
Model families carry traits across generations. Fine-tunes inherit strengths and weaknesses. Know the pedigree before you commit.
|
|
93
|
+
|
|
94
|
+
### 3. Respect the Fence
|
|
95
|
+
Conservation compliance — how well a model respects guardrails and bytecode fences — is non-negotiable for production work. A model that won't respect boundaries is a liability.
|
|
96
|
+
|
|
97
|
+
### 4. Trainability Matters
|
|
98
|
+
Some breeds take to fine-tuning naturally. Others resist. If you need to specialize, choose a breed known for trainability.
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## Adding a Breed
|
|
103
|
+
|
|
104
|
+
To register a new model:
|
|
105
|
+
|
|
106
|
+
1. Create `registry/<model-name>.json` following the assessment schema
|
|
107
|
+
2. Add an entry to `registry/index.json`
|
|
108
|
+
3. Submit a pull request
|
|
109
|
+
|
|
110
|
+
Assess honestly. Overrating a model's working aptitude helps no one — least of all the teams who deploy it.
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
## License
|
|
115
|
+
|
|
116
|
+
MIT
|
|
117
|
+
|
|
118
|
+
---
|
|
119
|
+
|
|
120
|
+
*Part of the [Working Animal Architecture](https://github.com/SuperInstance) — foundation models as working animals, not pets.*
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/breed_registry/__init__.py
|
|
5
|
+
src/breed_registry/matcher.py
|
|
6
|
+
src/breed_registry/models.py
|
|
7
|
+
src/breed_registry.egg-info/PKG-INFO
|
|
8
|
+
src/breed_registry.egg-info/SOURCES.txt
|
|
9
|
+
src/breed_registry.egg-info/dependency_links.txt
|
|
10
|
+
src/breed_registry.egg-info/top_level.txt
|
|
11
|
+
tests/test_matcher.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
breed_registry
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Tests for the Breed Registry matcher.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
from breed_registry import (
|
|
8
|
+
select_breed,
|
|
9
|
+
compare_breeds,
|
|
10
|
+
assess_aptitude,
|
|
11
|
+
list_breeds,
|
|
12
|
+
get_breed,
|
|
13
|
+
)
|
|
14
|
+
from breed_registry.models import AptitudeScore, ComparisonReport, ModelAssessment
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# ---------------------------------------------------------------------------
|
|
18
|
+
# Registry tests
|
|
19
|
+
# ---------------------------------------------------------------------------
|
|
20
|
+
|
|
21
|
+
class TestListBreeds:
|
|
22
|
+
def test_returns_all_registered_breeds(self):
|
|
23
|
+
breeds = list_breeds()
|
|
24
|
+
assert "gpt-4" in breeds
|
|
25
|
+
assert "claude-3" in breeds
|
|
26
|
+
assert "llama-3" in breeds
|
|
27
|
+
assert "glm" in breeds
|
|
28
|
+
assert "mistral" in breeds
|
|
29
|
+
|
|
30
|
+
def test_returns_sorted(self):
|
|
31
|
+
breeds = list_breeds()
|
|
32
|
+
assert breeds == sorted(breeds)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class TestGetBreed:
|
|
36
|
+
def test_returns_assessment(self):
|
|
37
|
+
assessment = get_breed("gpt-4")
|
|
38
|
+
assert assessment.name == "gpt-4"
|
|
39
|
+
assert assessment.breed_group == "General Purpose"
|
|
40
|
+
assert "code_generation" in assessment.working_aptitude
|
|
41
|
+
|
|
42
|
+
def test_raises_for_unknown_breed(self):
|
|
43
|
+
with pytest.raises(KeyError, match="not registered"):
|
|
44
|
+
get_breed("nonexistent-model")
|
|
45
|
+
|
|
46
|
+
def test_assessment_has_all_fields(self):
|
|
47
|
+
assessment = get_breed("claude-3")
|
|
48
|
+
assert assessment.lineage
|
|
49
|
+
assert assessment.temperament
|
|
50
|
+
assert assessment.cost_profile
|
|
51
|
+
assert assessment.speed_profile
|
|
52
|
+
assert assessment.trainability
|
|
53
|
+
assert assessment.recommended_for
|
|
54
|
+
assert assessment.not_recommended_for
|
|
55
|
+
assert assessment.fence_compatibility
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
# ---------------------------------------------------------------------------
|
|
59
|
+
# select_breed tests
|
|
60
|
+
# ---------------------------------------------------------------------------
|
|
61
|
+
|
|
62
|
+
class TestSelectBreed:
|
|
63
|
+
def test_returns_recommendations(self):
|
|
64
|
+
recommendations = select_breed("code_generation")
|
|
65
|
+
assert len(recommendations) > 0
|
|
66
|
+
assert all(isinstance(r, ModelAssessment) for r in recommendations)
|
|
67
|
+
|
|
68
|
+
def test_sorted_by_aptitude(self):
|
|
69
|
+
recommendations = select_breed("analysis", top_k=5)
|
|
70
|
+
scores = [r.aptitude_for("analysis") for r in recommendations]
|
|
71
|
+
assert scores == sorted(scores, reverse=True)
|
|
72
|
+
|
|
73
|
+
def test_respects_top_k(self):
|
|
74
|
+
recommendations = select_breed("code_generation", top_k=2)
|
|
75
|
+
assert len(recommendations) <= 2
|
|
76
|
+
|
|
77
|
+
def test_gpt4_top_for_code_generation(self):
|
|
78
|
+
recommendations = select_breed("code_generation", top_k=1)
|
|
79
|
+
assert recommendations[0].name == "gpt-4"
|
|
80
|
+
|
|
81
|
+
def test_claude3_top_for_analysis(self):
|
|
82
|
+
recommendations = select_breed("analysis", top_k=1)
|
|
83
|
+
assert recommendations[0].name == "claude-3"
|
|
84
|
+
|
|
85
|
+
def test_cost_filter_free(self):
|
|
86
|
+
recommendations = select_breed("code_generation", max_cost="free")
|
|
87
|
+
for r in recommendations:
|
|
88
|
+
assert r.cost_profile == "free"
|
|
89
|
+
|
|
90
|
+
def test_cost_filter_low(self):
|
|
91
|
+
recommendations = select_breed("code_generation", max_cost="low")
|
|
92
|
+
for r in recommendations:
|
|
93
|
+
assert r.cost_profile in ("free", "low")
|
|
94
|
+
|
|
95
|
+
def test_unknown_task_returns_empty(self):
|
|
96
|
+
recommendations = select_breed("nonexistent_task")
|
|
97
|
+
assert len(recommendations) == 0
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# ---------------------------------------------------------------------------
|
|
101
|
+
# compare_breeds tests
|
|
102
|
+
# ---------------------------------------------------------------------------
|
|
103
|
+
|
|
104
|
+
class TestCompareBreeds:
|
|
105
|
+
def test_returns_comparison_report(self):
|
|
106
|
+
report = compare_breeds("gpt-4", "llama-3")
|
|
107
|
+
assert isinstance(report, ComparisonReport)
|
|
108
|
+
assert report.model_a == "gpt-4"
|
|
109
|
+
assert report.model_b == "llama-3"
|
|
110
|
+
|
|
111
|
+
def test_comparison_has_all_tasks(self):
|
|
112
|
+
report = compare_breeds("gpt-4", "claude-3")
|
|
113
|
+
breed_a = get_breed("gpt-4")
|
|
114
|
+
breed_b = get_breed("claude-3")
|
|
115
|
+
all_tasks = set(breed_a.working_aptitude.keys()) | set(breed_b.working_aptitude.keys())
|
|
116
|
+
assert set(report.aptitude_comparison.keys()) == all_tasks
|
|
117
|
+
|
|
118
|
+
def test_winner_is_determined(self):
|
|
119
|
+
report = compare_breeds("gpt-4", "llama-3")
|
|
120
|
+
assert report.winner in ("gpt-4", "llama-3", "tie")
|
|
121
|
+
assert report.margin >= 0
|
|
122
|
+
|
|
123
|
+
def test_gpt4_vs_llama3_gpt4_wins(self):
|
|
124
|
+
report = compare_breeds("gpt-4", "llama-3")
|
|
125
|
+
assert report.winner == "gpt-4"
|
|
126
|
+
|
|
127
|
+
def test_advantages_populated(self):
|
|
128
|
+
report = compare_breeds("gpt-4", "llama-3")
|
|
129
|
+
# GPT-4 should have advantages in at least some tasks
|
|
130
|
+
assert len(report.advantages_a) > 0 or len(report.advantages_b) > 0
|
|
131
|
+
|
|
132
|
+
def test_summary_method_works(self):
|
|
133
|
+
report = compare_breeds("gpt-4", "claude-3")
|
|
134
|
+
summary = report.summary()
|
|
135
|
+
assert "gpt-4" in summary
|
|
136
|
+
assert "claude-3" in summary
|
|
137
|
+
assert "winner" in summary.lower()
|
|
138
|
+
|
|
139
|
+
def test_raises_for_unknown_model(self):
|
|
140
|
+
with pytest.raises(KeyError):
|
|
141
|
+
compare_breeds("gpt-4", "nonexistent")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
# ---------------------------------------------------------------------------
|
|
145
|
+
# assess_aptitude tests
|
|
146
|
+
# ---------------------------------------------------------------------------
|
|
147
|
+
|
|
148
|
+
class TestAssessAptitude:
|
|
149
|
+
def test_returns_aptitude_score(self):
|
|
150
|
+
score = assess_aptitude("mistral", "analysis")
|
|
151
|
+
assert isinstance(score, AptitudeScore)
|
|
152
|
+
assert score.model == "mistral"
|
|
153
|
+
assert score.task == "analysis"
|
|
154
|
+
assert 0 <= score.score <= 10
|
|
155
|
+
|
|
156
|
+
def test_rating_matches_score(self):
|
|
157
|
+
score = assess_aptitude("gpt-4", "code_generation")
|
|
158
|
+
assert score.score == 9
|
|
159
|
+
assert score.rating == "excellent"
|
|
160
|
+
|
|
161
|
+
def test_percentile_calculated(self):
|
|
162
|
+
score = assess_aptitude("gpt-4", "code_generation")
|
|
163
|
+
assert score.percentile is not None
|
|
164
|
+
assert 0.0 <= score.percentile <= 100.0
|
|
165
|
+
|
|
166
|
+
def test_claude3_perfect_analysis(self):
|
|
167
|
+
score = assess_aptitude("claude-3", "analysis")
|
|
168
|
+
assert score.score == 10
|
|
169
|
+
|
|
170
|
+
def test_raises_for_unknown_model(self):
|
|
171
|
+
with pytest.raises(KeyError):
|
|
172
|
+
assess_aptitude("nonexistent", "code_generation")
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
# ---------------------------------------------------------------------------
|
|
176
|
+
# ModelAssessment unit tests
|
|
177
|
+
# ---------------------------------------------------------------------------
|
|
178
|
+
|
|
179
|
+
class TestModelAssessment:
|
|
180
|
+
def test_overall_score(self):
|
|
181
|
+
assessment = get_breed("gpt-4")
|
|
182
|
+
overall = assessment.overall_score()
|
|
183
|
+
assert overall > 0
|
|
184
|
+
assert overall <= 10
|
|
185
|
+
|
|
186
|
+
def test_aptitude_for_known_task(self):
|
|
187
|
+
assessment = get_breed("gpt-4")
|
|
188
|
+
assert assessment.aptitude_for("code_generation") == 9
|
|
189
|
+
|
|
190
|
+
def test_aptitude_for_unknown_task(self):
|
|
191
|
+
assessment = get_breed("gpt-4")
|
|
192
|
+
assert assessment.aptitude_for("nonexistent") == 0
|
|
193
|
+
|
|
194
|
+
def test_from_dict_roundtrip(self):
|
|
195
|
+
original = get_breed("gpt-4")
|
|
196
|
+
data = {
|
|
197
|
+
"name": original.name,
|
|
198
|
+
"lineage": original.lineage,
|
|
199
|
+
"breed_group": original.breed_group,
|
|
200
|
+
"temperament": original.temperament,
|
|
201
|
+
"working_aptitude": original.working_aptitude,
|
|
202
|
+
"cost_profile": original.cost_profile,
|
|
203
|
+
"speed_profile": original.speed_profile,
|
|
204
|
+
"trainability": original.trainability,
|
|
205
|
+
"recommended_for": original.recommended_for,
|
|
206
|
+
"not_recommended_for": original.not_recommended_for,
|
|
207
|
+
"fence_compatibility": original.fence_compatibility,
|
|
208
|
+
"notes": original.notes,
|
|
209
|
+
}
|
|
210
|
+
rebuilt = ModelAssessment.from_dict(data)
|
|
211
|
+
assert rebuilt.name == original.name
|
|
212
|
+
assert rebuilt.overall_score() == original.overall_score()
|