dialectus-engine 0.3.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dialectus_engine-0.3.0/dialectus_engine.egg-info → dialectus_engine-0.3.1}/PKG-INFO +1 -1
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/__init__.py +1 -1
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/config/settings.py +41 -16
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/context_builder.py +11 -7
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/engine.py +8 -11
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/round_manager.py +23 -23
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/types.py +8 -6
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/utils.py +11 -6
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/judges/ai_judge.py +20 -9
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/judges/base.py +32 -43
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/judges/ensemble_utils.py +44 -29
- dialectus_engine-0.3.1/dialectus/engine/models/__init__.py +17 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/manager.py +12 -8
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_filter_config.py +3 -1
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/providers/base_model_provider.py +31 -9
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/providers/ollama_provider.py +7 -7
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/providers/open_router_provider.py +93 -53
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/providers/openrouter_generation_types.py +44 -3
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/moderation/openai_moderator.py +20 -55
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1/dialectus_engine.egg-info}/PKG-INFO +1 -1
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/pyproject.toml +2 -1
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/tests/test_moderation_openai.py +39 -6
- dialectus_engine-0.3.0/dialectus/engine/models/__init__.py +0 -1
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/LICENSE +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/README.md +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/__init__.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/config/__init__.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/config/openrouter_filters.json +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/__init__.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/judge_coordinator.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/models.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/prompt_builder.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/response_handler.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/formats/__init__.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/formats/base.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/formats/oxford.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/formats/parliamentary.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/formats/public_forum.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/formats/registry.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/formats/socratic.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/formats/topic_themes.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/judges/__init__.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/judges/factory.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/base_types.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/cache_manager.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_architecture.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_capability_extractor.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_enhanced_model_info.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_model.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_model_filter.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_models_response.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_pricing.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_top_provider.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/providers/__init__.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/providers/exceptions.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/providers/providers.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/moderation/__init__.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/moderation/base_moderator.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/moderation/exceptions.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/moderation/llm_moderator.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/moderation/manager.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus_engine.egg-info/SOURCES.txt +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus_engine.egg-info/dependency_links.txt +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus_engine.egg-info/requires.txt +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus_engine.egg-info/top_level.txt +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/setup.cfg +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/tests/test_cache_manager.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/tests/test_config_settings.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/tests/test_debate_models.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/tests/test_format_registry.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/tests/test_formats_base.py +0 -0
- {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/tests/test_judges_factory.py +0 -0
|
@@ -6,6 +6,25 @@ from typing import Literal
|
|
|
6
6
|
from pydantic import BaseModel, Field, field_validator
|
|
7
7
|
|
|
8
8
|
|
|
9
|
+
def validate_provider_value(value: str, valid_providers: set[str]) -> str:
|
|
10
|
+
"""Validate provider string against allowed values.
|
|
11
|
+
|
|
12
|
+
Args:
|
|
13
|
+
value: Provider name to validate
|
|
14
|
+
valid_providers: Set of valid provider names
|
|
15
|
+
|
|
16
|
+
Returns:
|
|
17
|
+
Lowercase provider name
|
|
18
|
+
|
|
19
|
+
Raises:
|
|
20
|
+
ValueError: If provider is not in valid_providers
|
|
21
|
+
"""
|
|
22
|
+
normalized = value.lower()
|
|
23
|
+
if normalized not in valid_providers:
|
|
24
|
+
raise ValueError(f"Provider must be one of: {valid_providers}")
|
|
25
|
+
return normalized
|
|
26
|
+
|
|
27
|
+
|
|
9
28
|
class ModelConfig(BaseModel):
|
|
10
29
|
"""Configuration for a debate model."""
|
|
11
30
|
|
|
@@ -24,11 +43,8 @@ class ModelConfig(BaseModel):
|
|
|
24
43
|
|
|
25
44
|
@field_validator("provider")
|
|
26
45
|
@classmethod
|
|
27
|
-
def validate_provider(cls, v: str):
|
|
28
|
-
|
|
29
|
-
if v not in valid_providers:
|
|
30
|
-
raise ValueError(f"Provider must be one of: {valid_providers}")
|
|
31
|
-
return v
|
|
46
|
+
def validate_provider(cls, v: str) -> str:
|
|
47
|
+
return validate_provider_value(v, {"ollama", "openrouter"})
|
|
32
48
|
|
|
33
49
|
|
|
34
50
|
class DebateConfig(BaseModel):
|
|
@@ -93,9 +109,9 @@ class ModerationConfig(BaseModel):
|
|
|
93
109
|
|
|
94
110
|
@field_validator("provider")
|
|
95
111
|
@classmethod
|
|
96
|
-
def validate_provider(cls, v: str):
|
|
112
|
+
def validate_provider(cls, v: str) -> str:
|
|
97
113
|
# Provider names are case-insensitive
|
|
98
|
-
#
|
|
114
|
+
# For moderation, we accept any provider (ollama, openrouter, openai, custom)
|
|
99
115
|
# Custom providers require explicit base_url
|
|
100
116
|
return v.lower()
|
|
101
117
|
|
|
@@ -183,11 +199,8 @@ class SystemConfig(BaseModel):
|
|
|
183
199
|
|
|
184
200
|
@field_validator("debate_topic_source")
|
|
185
201
|
@classmethod
|
|
186
|
-
def validate_topic_source(cls, v: str):
|
|
187
|
-
|
|
188
|
-
if v not in valid_sources:
|
|
189
|
-
raise ValueError(f"debate_topic_source must be one of: {valid_sources}")
|
|
190
|
-
return v
|
|
202
|
+
def validate_topic_source(cls, v: str) -> str:
|
|
203
|
+
return validate_provider_value(v, {"ollama", "openrouter"})
|
|
191
204
|
|
|
192
205
|
|
|
193
206
|
class AppConfig(BaseModel):
|
|
@@ -203,14 +216,26 @@ class AppConfig(BaseModel):
|
|
|
203
216
|
|
|
204
217
|
@classmethod
|
|
205
218
|
def load_from_file(cls, config_path: Path) -> "AppConfig":
|
|
206
|
-
"""Load configuration from JSON file.
|
|
219
|
+
"""Load configuration from JSON file.
|
|
220
|
+
|
|
221
|
+
Args:
|
|
222
|
+
config_path: Path to the JSON configuration file
|
|
223
|
+
|
|
224
|
+
Returns:
|
|
225
|
+
AppConfig instance loaded from the file
|
|
226
|
+
|
|
227
|
+
Raises:
|
|
228
|
+
FileNotFoundError: If config file doesn't exist
|
|
229
|
+
ValueError: If required sections are missing or invalid
|
|
230
|
+
"""
|
|
207
231
|
if not config_path.exists():
|
|
208
232
|
raise FileNotFoundError(f"Config file not found: {config_path}")
|
|
209
233
|
|
|
234
|
+
# Use Pydantic's built-in JSON validation
|
|
210
235
|
import json
|
|
211
236
|
|
|
212
|
-
|
|
213
|
-
|
|
237
|
+
json_data = config_path.read_text(encoding="utf-8")
|
|
238
|
+
data = json.loads(json_data)
|
|
214
239
|
|
|
215
240
|
# Validate required sections
|
|
216
241
|
required_sections = ["debate", "models", "judging", "system"]
|
|
@@ -226,7 +251,7 @@ class AppConfig(BaseModel):
|
|
|
226
251
|
"Config must include at least one model in 'models' section"
|
|
227
252
|
)
|
|
228
253
|
|
|
229
|
-
return cls(
|
|
254
|
+
return cls.model_validate(data)
|
|
230
255
|
|
|
231
256
|
|
|
232
257
|
def get_default_config() -> AppConfig:
|
{dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/context_builder.py
RENAMED
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
from typing import TYPE_CHECKING
|
|
5
|
+
from typing import TYPE_CHECKING, cast
|
|
6
6
|
|
|
7
7
|
from dialectus.engine.config.settings import AppConfig
|
|
8
|
+
from dialectus.engine.models.providers.base_model_provider import ChatMessage
|
|
8
9
|
|
|
9
10
|
from .models import DebateContext
|
|
10
11
|
from .prompt_builder import PromptBuilder
|
|
@@ -13,6 +14,9 @@ from .types import DebatePhase
|
|
|
13
14
|
if TYPE_CHECKING:
|
|
14
15
|
from dialectus.engine.formats import DebateFormat, FormatPhase
|
|
15
16
|
|
|
17
|
+
# Maximum number of recent messages to include in conversation context
|
|
18
|
+
MAX_CONTEXT_MESSAGES = 6
|
|
19
|
+
|
|
16
20
|
|
|
17
21
|
class ContextBuilder:
|
|
18
22
|
"""Builds conversation context for model prompts during debates."""
|
|
@@ -39,7 +43,7 @@ class ContextBuilder:
|
|
|
39
43
|
speaker_id: str,
|
|
40
44
|
phase: DebatePhase,
|
|
41
45
|
context: DebateContext,
|
|
42
|
-
) -> list[
|
|
46
|
+
) -> list[ChatMessage]:
|
|
43
47
|
"""Build conversation context for the model.
|
|
44
48
|
|
|
45
49
|
Args:
|
|
@@ -71,7 +75,7 @@ class ContextBuilder:
|
|
|
71
75
|
})
|
|
72
76
|
|
|
73
77
|
# Add relevant debate history
|
|
74
|
-
for msg in context.messages[-
|
|
78
|
+
for msg in context.messages[-MAX_CONTEXT_MESSAGES:]:
|
|
75
79
|
role = "assistant" if msg.speaker_id == speaker_id else "user"
|
|
76
80
|
|
|
77
81
|
# Use a cleaner format that's less likely to be repeated by small
|
|
@@ -94,14 +98,14 @@ class ContextBuilder:
|
|
|
94
98
|
)
|
|
95
99
|
messages.append({"role": "user", "content": turn_prompt})
|
|
96
100
|
|
|
97
|
-
return messages
|
|
101
|
+
return cast(list[ChatMessage], messages)
|
|
98
102
|
|
|
99
103
|
def build_format_conversation_context(
|
|
100
104
|
self,
|
|
101
105
|
speaker_id: str,
|
|
102
106
|
format_phase: FormatPhase,
|
|
103
107
|
context: DebateContext,
|
|
104
|
-
) -> list[
|
|
108
|
+
) -> list[ChatMessage]:
|
|
105
109
|
"""Build conversation context for the model using format phase.
|
|
106
110
|
|
|
107
111
|
Args:
|
|
@@ -139,7 +143,7 @@ class ContextBuilder:
|
|
|
139
143
|
})
|
|
140
144
|
|
|
141
145
|
# Add relevant debate history
|
|
142
|
-
for msg in context.messages[-
|
|
146
|
+
for msg in context.messages[-MAX_CONTEXT_MESSAGES:]:
|
|
143
147
|
role = "assistant" if msg.speaker_id == speaker_id else "user"
|
|
144
148
|
|
|
145
149
|
# Use a cleaner format that's less likely to be repeated by small
|
|
@@ -160,4 +164,4 @@ class ContextBuilder:
|
|
|
160
164
|
turn_prompt = f"Now speak as {role_name}. Stay under {word_limit} words."
|
|
161
165
|
messages.append({"role": "user", "content": turn_prompt})
|
|
162
166
|
|
|
163
|
-
return messages
|
|
167
|
+
return cast(list[ChatMessage], messages)
|
|
@@ -22,6 +22,7 @@ from .types import (
|
|
|
22
22
|
MessageEventCallback,
|
|
23
23
|
PhaseEventCallback,
|
|
24
24
|
PhaseEventType,
|
|
25
|
+
PhaseStartedEventData,
|
|
25
26
|
)
|
|
26
27
|
|
|
27
28
|
logger = logging.getLogger(__name__)
|
|
@@ -169,18 +170,14 @@ class DebateEngine:
|
|
|
169
170
|
|
|
170
171
|
# Invoke phase callback if provided
|
|
171
172
|
if phase_callback:
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
"total_phases": total_phases,
|
|
179
|
-
"progress_percentage": round(
|
|
180
|
-
(phase_index / total_phases) * 100
|
|
181
|
-
),
|
|
182
|
-
},
|
|
173
|
+
phase_event = PhaseStartedEventData(
|
|
174
|
+
phase=format_phase.name,
|
|
175
|
+
instruction=format_phase.instruction,
|
|
176
|
+
current_phase=self.context.current_round,
|
|
177
|
+
total_phases=total_phases,
|
|
178
|
+
progress_percentage=round((phase_index / total_phases) * 100),
|
|
183
179
|
)
|
|
180
|
+
await phase_callback(PhaseEventType.PHASE_STARTED, phase_event)
|
|
184
181
|
|
|
185
182
|
# If streaming callbacks provided, use streaming version
|
|
186
183
|
if message_callback or chunk_callback:
|
{dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/round_manager.py
RENAMED
|
@@ -11,8 +11,10 @@ from .response_handler import ResponseHandler
|
|
|
11
11
|
from .types import (
|
|
12
12
|
ChunkCallback,
|
|
13
13
|
DebatePhase,
|
|
14
|
+
MessageCompleteEventData,
|
|
14
15
|
MessageEventCallback,
|
|
15
16
|
MessageEventType,
|
|
17
|
+
MessageStartEventData,
|
|
16
18
|
Position,
|
|
17
19
|
)
|
|
18
20
|
|
|
@@ -129,16 +131,14 @@ class RoundManager:
|
|
|
129
131
|
|
|
130
132
|
# Call message_start callback
|
|
131
133
|
if message_callback:
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
"phase": format_phase.phase.value,
|
|
139
|
-
"round_number": context.current_round,
|
|
140
|
-
},
|
|
134
|
+
start_event = MessageStartEventData(
|
|
135
|
+
message_id=message_id,
|
|
136
|
+
speaker_id=speaker_id,
|
|
137
|
+
position=speaker_position.value,
|
|
138
|
+
phase=format_phase.phase.value,
|
|
139
|
+
round_number=context.current_round,
|
|
141
140
|
)
|
|
141
|
+
await message_callback(MessageEventType.MESSAGE_START, start_event)
|
|
142
142
|
|
|
143
143
|
# Create streaming wrapper that calls chunk_callback
|
|
144
144
|
async def chunk_wrapper(chunk: str, is_complete: bool):
|
|
@@ -154,21 +154,21 @@ class RoundManager:
|
|
|
154
154
|
|
|
155
155
|
# Call message_complete callback
|
|
156
156
|
if message_callback:
|
|
157
|
+
complete_event = MessageCompleteEventData(
|
|
158
|
+
message_id=message_id,
|
|
159
|
+
speaker_id=message.speaker_id,
|
|
160
|
+
position=message.position.value,
|
|
161
|
+
phase=message.phase.value,
|
|
162
|
+
round_number=message.round_number,
|
|
163
|
+
content=message.content,
|
|
164
|
+
timestamp=message.timestamp.isoformat(),
|
|
165
|
+
word_count=len(message.content.split()),
|
|
166
|
+
metadata=message.metadata,
|
|
167
|
+
cost=message.cost,
|
|
168
|
+
generation_id=message.generation_id,
|
|
169
|
+
)
|
|
157
170
|
await message_callback(
|
|
158
|
-
MessageEventType.MESSAGE_COMPLETE,
|
|
159
|
-
{
|
|
160
|
-
"message_id": message_id,
|
|
161
|
-
"speaker_id": message.speaker_id,
|
|
162
|
-
"position": message.position.value,
|
|
163
|
-
"phase": message.phase.value,
|
|
164
|
-
"round_number": message.round_number,
|
|
165
|
-
"content": message.content,
|
|
166
|
-
"timestamp": message.timestamp.isoformat(),
|
|
167
|
-
"word_count": len(message.content.split()),
|
|
168
|
-
"metadata": message.metadata,
|
|
169
|
-
"cost": message.cost,
|
|
170
|
-
"generation_id": message.generation_id,
|
|
171
|
-
},
|
|
171
|
+
MessageEventType.MESSAGE_COMPLETE, complete_event
|
|
172
172
|
)
|
|
173
173
|
|
|
174
174
|
logger.info(
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
from collections.abc import Awaitable, Callable
|
|
4
4
|
from enum import Enum
|
|
5
|
-
from typing import Protocol
|
|
5
|
+
from typing import Protocol
|
|
6
|
+
|
|
7
|
+
from pydantic import BaseModel
|
|
6
8
|
|
|
7
9
|
|
|
8
10
|
class PhaseEventType(str, Enum):
|
|
@@ -19,7 +21,7 @@ class MessageEventType(str, Enum):
|
|
|
19
21
|
MESSAGE_COMPLETE = "message_complete"
|
|
20
22
|
|
|
21
23
|
|
|
22
|
-
class PhaseStartedEventData(
|
|
24
|
+
class PhaseStartedEventData(BaseModel):
|
|
23
25
|
"""Data structure for phase_started event callbacks."""
|
|
24
26
|
|
|
25
27
|
phase: str
|
|
@@ -29,7 +31,7 @@ class PhaseStartedEventData(TypedDict):
|
|
|
29
31
|
progress_percentage: int
|
|
30
32
|
|
|
31
33
|
|
|
32
|
-
class MessageStartEventData(
|
|
34
|
+
class MessageStartEventData(BaseModel):
|
|
33
35
|
"""Data structure for message_start event callbacks."""
|
|
34
36
|
|
|
35
37
|
message_id: str
|
|
@@ -39,7 +41,7 @@ class MessageStartEventData(TypedDict):
|
|
|
39
41
|
round_number: int
|
|
40
42
|
|
|
41
43
|
|
|
42
|
-
class MessageCompleteEventData(
|
|
44
|
+
class MessageCompleteEventData(BaseModel):
|
|
43
45
|
"""Data structure for message_complete event callbacks."""
|
|
44
46
|
|
|
45
47
|
message_id: str
|
|
@@ -51,8 +53,8 @@ class MessageCompleteEventData(TypedDict):
|
|
|
51
53
|
timestamp: str
|
|
52
54
|
word_count: int
|
|
53
55
|
metadata: dict[str, str]
|
|
54
|
-
cost: float | None
|
|
55
|
-
generation_id: str | None
|
|
56
|
+
cost: float | None = None
|
|
57
|
+
generation_id: str | None = None
|
|
56
58
|
|
|
57
59
|
|
|
58
60
|
# Callback protocols with proper type narrowing
|
|
@@ -10,6 +10,14 @@ from .models import DebateMessage
|
|
|
10
10
|
|
|
11
11
|
logger = logging.getLogger(__name__)
|
|
12
12
|
|
|
13
|
+
# Token calculation constants
|
|
14
|
+
# Average tokens per word ratio (1 token ≈ 0.75 words, so multiply words by 1.33)
|
|
15
|
+
TOKENS_PER_WORD_RATIO = 1.33
|
|
16
|
+
# Buffer for sentence completion in words
|
|
17
|
+
SENTENCE_BUFFER_WORDS = 100
|
|
18
|
+
# Buffer for sentence completion in tokens (100 words * 1.33)
|
|
19
|
+
SENTENCE_BUFFER_TOKENS = 133
|
|
20
|
+
|
|
13
21
|
|
|
14
22
|
def calculate_max_tokens(word_limit: int) -> int:
|
|
15
23
|
"""Calculate maximum tokens based on word limit with buffer for incomplete
|
|
@@ -19,13 +27,10 @@ def calculate_max_tokens(word_limit: int) -> int:
|
|
|
19
27
|
word_limit: Maximum number of words allowed
|
|
20
28
|
|
|
21
29
|
Returns:
|
|
22
|
-
Maximum number of tokens with
|
|
30
|
+
Maximum number of tokens with buffer for sentence completion
|
|
23
31
|
"""
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
base_tokens = int(word_limit * 1.33)
|
|
27
|
-
buffer_tokens = 133 # ~100 words of buffer for sentence completion
|
|
28
|
-
return base_tokens + buffer_tokens
|
|
32
|
+
base_tokens = int(word_limit * TOKENS_PER_WORD_RATIO)
|
|
33
|
+
return base_tokens + SENTENCE_BUFFER_TOKENS
|
|
29
34
|
|
|
30
35
|
|
|
31
36
|
def trim_incomplete_sentence(text: str) -> str:
|
|
@@ -8,10 +8,12 @@ import time
|
|
|
8
8
|
import uuid
|
|
9
9
|
from datetime import datetime
|
|
10
10
|
from textwrap import dedent
|
|
11
|
+
from typing import cast
|
|
11
12
|
|
|
12
13
|
from dialectus.engine.config.settings import SystemConfig
|
|
13
14
|
from dialectus.engine.debate_engine.models import DebateContext
|
|
14
15
|
from dialectus.engine.models.manager import ModelManager
|
|
16
|
+
from dialectus.engine.models.providers.base_model_provider import ChatMessage
|
|
15
17
|
|
|
16
18
|
from .base import BaseJudge, CriterionScore, JudgeDecision, JudgmentCriterion
|
|
17
19
|
|
|
@@ -38,6 +40,12 @@ class AIJudge(BaseJudge):
|
|
|
38
40
|
# Initialize ensemble temperature (can be overridden for ensemble judges)
|
|
39
41
|
self._ensemble_temperature: float = 0.3
|
|
40
42
|
|
|
43
|
+
# Generation metadata - tracked per evaluation
|
|
44
|
+
self._last_generation_time_ms: int | None = None
|
|
45
|
+
self._last_generation_cost: float | None = None
|
|
46
|
+
self._last_generation_id: str | None = None
|
|
47
|
+
self._last_generation_provider: str | None = None
|
|
48
|
+
|
|
41
49
|
# Generate unique judge ID for model manager registration
|
|
42
50
|
self.judge_id = str(uuid.uuid4())
|
|
43
51
|
|
|
@@ -166,17 +174,20 @@ class AIJudge(BaseJudge):
|
|
|
166
174
|
transcript, participants, criteria_list, context
|
|
167
175
|
)
|
|
168
176
|
|
|
169
|
-
messages =
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
177
|
+
messages: list[ChatMessage] = cast(
|
|
178
|
+
list[ChatMessage],
|
|
179
|
+
[
|
|
180
|
+
{"role": "system", "content": self._get_judge_system_prompt()},
|
|
181
|
+
{"role": "user", "content": evaluation_prompt},
|
|
182
|
+
],
|
|
183
|
+
)
|
|
173
184
|
|
|
174
185
|
# Capture evaluation timing
|
|
175
186
|
start_time = time.time()
|
|
176
187
|
|
|
177
188
|
# Use model manager to generate response
|
|
178
189
|
# Add some randomness for ensemble judges to get different evaluations
|
|
179
|
-
temperature =
|
|
190
|
+
temperature = self._ensemble_temperature
|
|
180
191
|
|
|
181
192
|
async with self.model_manager.model_session(self.judge_id):
|
|
182
193
|
generation_metadata = (
|
|
@@ -459,15 +470,15 @@ parsed programmatically. Be thorough but concise in your reasoning."""
|
|
|
459
470
|
reasoning=reasoning,
|
|
460
471
|
judge_model=self.judge_model_name,
|
|
461
472
|
judge_provider=self.judge_provider,
|
|
462
|
-
generation_time_ms=
|
|
463
|
-
cost=
|
|
464
|
-
generation_id=
|
|
473
|
+
generation_time_ms=self._last_generation_time_ms,
|
|
474
|
+
cost=self._last_generation_cost,
|
|
475
|
+
generation_id=self._last_generation_id,
|
|
465
476
|
)
|
|
466
477
|
|
|
467
478
|
# Schedule background cost query for OpenRouter judges
|
|
468
479
|
if (
|
|
469
480
|
judge_decision.generation_id
|
|
470
|
-
and
|
|
481
|
+
and self._last_generation_provider == "openrouter"
|
|
471
482
|
):
|
|
472
483
|
asyncio.create_task(self._query_and_update_judge_cost(judge_decision))
|
|
473
484
|
|
|
@@ -62,6 +62,35 @@ class BaseJudge(ABC):
|
|
|
62
62
|
"""Judge name/identifier."""
|
|
63
63
|
pass
|
|
64
64
|
|
|
65
|
+
def _get_participant_averages(
|
|
66
|
+
self, criterion_scores: list[CriterionScore]
|
|
67
|
+
) -> dict[str, float]:
|
|
68
|
+
"""Calculate average scores for each participant across all criteria.
|
|
69
|
+
|
|
70
|
+
Args:
|
|
71
|
+
criterion_scores: List of criterion scores from the debate
|
|
72
|
+
|
|
73
|
+
Returns:
|
|
74
|
+
Dictionary mapping participant IDs to their average scores
|
|
75
|
+
"""
|
|
76
|
+
participant_totals: dict[str, float] = {}
|
|
77
|
+
participant_counts: dict[str, int] = {}
|
|
78
|
+
|
|
79
|
+
for score in criterion_scores:
|
|
80
|
+
participant_id = score.participant_id
|
|
81
|
+
participant_totals[participant_id] = (
|
|
82
|
+
participant_totals.get(participant_id, 0.0) + score.score
|
|
83
|
+
)
|
|
84
|
+
participant_counts[participant_id] = (
|
|
85
|
+
participant_counts.get(participant_id, 0) + 1
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
return {
|
|
89
|
+
pid: participant_totals[pid] / participant_counts[pid]
|
|
90
|
+
for pid in participant_totals
|
|
91
|
+
if participant_counts[pid] > 0
|
|
92
|
+
}
|
|
93
|
+
|
|
65
94
|
def _calculate_total_score(
|
|
66
95
|
self, scores: list[CriterionScore], participant_id: str
|
|
67
96
|
) -> float:
|
|
@@ -80,34 +109,13 @@ class BaseJudge(ABC):
|
|
|
80
109
|
if not criterion_scores:
|
|
81
110
|
return 0.0
|
|
82
111
|
|
|
83
|
-
|
|
84
|
-
participant_totals: dict[str, float] = {}
|
|
85
|
-
participant_counts: dict[str, int] = {}
|
|
86
|
-
|
|
87
|
-
for score in criterion_scores:
|
|
88
|
-
participant_id = score.participant_id
|
|
89
|
-
if participant_id not in participant_totals:
|
|
90
|
-
participant_totals[participant_id] = 0.0
|
|
91
|
-
participant_counts[participant_id] = 0
|
|
92
|
-
|
|
93
|
-
participant_totals[participant_id] += score.score
|
|
94
|
-
participant_counts[participant_id] += 1
|
|
95
|
-
|
|
96
|
-
# Calculate average scores for each participant
|
|
97
|
-
participant_averages: dict[str, float] = {}
|
|
98
|
-
for participant_id in participant_totals:
|
|
99
|
-
if participant_counts[participant_id] > 0:
|
|
100
|
-
participant_averages[participant_id] = (
|
|
101
|
-
participant_totals[participant_id]
|
|
102
|
-
/ participant_counts[participant_id]
|
|
103
|
-
)
|
|
112
|
+
participant_averages = self._get_participant_averages(criterion_scores)
|
|
104
113
|
|
|
105
114
|
# Find margin between highest and second highest
|
|
106
115
|
if len(participant_averages) < 2:
|
|
107
116
|
return 0.0
|
|
108
117
|
|
|
109
|
-
averages =
|
|
110
|
-
averages.sort(reverse=True)
|
|
118
|
+
averages = sorted(participant_averages.values(), reverse=True)
|
|
111
119
|
return averages[0] - averages[1]
|
|
112
120
|
|
|
113
121
|
def _determine_winner_from_scores(
|
|
@@ -117,26 +125,7 @@ class BaseJudge(ABC):
|
|
|
117
125
|
if not criterion_scores:
|
|
118
126
|
return "unknown"
|
|
119
127
|
|
|
120
|
-
participant_averages
|
|
121
|
-
participant_totals: dict[str, float] = {}
|
|
122
|
-
participant_counts: dict[str, int] = {}
|
|
123
|
-
|
|
124
|
-
for score in criterion_scores:
|
|
125
|
-
participant_id = score.participant_id
|
|
126
|
-
if participant_id not in participant_totals:
|
|
127
|
-
participant_totals[participant_id] = 0.0
|
|
128
|
-
participant_counts[participant_id] = 0
|
|
129
|
-
|
|
130
|
-
participant_totals[participant_id] += score.score
|
|
131
|
-
participant_counts[participant_id] += 1
|
|
132
|
-
|
|
133
|
-
# Calculate averages
|
|
134
|
-
for participant_id in participant_totals:
|
|
135
|
-
if participant_counts[participant_id] > 0:
|
|
136
|
-
participant_averages[participant_id] = (
|
|
137
|
-
participant_totals[participant_id]
|
|
138
|
-
/ participant_counts[participant_id]
|
|
139
|
-
)
|
|
128
|
+
participant_averages = self._get_participant_averages(criterion_scores)
|
|
140
129
|
|
|
141
130
|
# Return participant with highest average
|
|
142
131
|
return (
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"""Ensemble judging utilities - standalone functions for combining judge decisions."""
|
|
2
2
|
|
|
3
3
|
import logging
|
|
4
|
-
from
|
|
4
|
+
from dataclasses import dataclass
|
|
5
5
|
|
|
6
6
|
from dialectus.engine.debate_engine.models import DebateContext
|
|
7
7
|
|
|
@@ -10,7 +10,8 @@ from .base import CriterionScore, JudgeDecision
|
|
|
10
10
|
logger = logging.getLogger(__name__)
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
@dataclass
|
|
14
|
+
class SerializedCriterionScore:
|
|
14
15
|
"""Serialized criterion score for storage."""
|
|
15
16
|
|
|
16
17
|
criterion: str
|
|
@@ -19,7 +20,8 @@ class SerializedCriterionScore(TypedDict):
|
|
|
19
20
|
feedback: str
|
|
20
21
|
|
|
21
22
|
|
|
22
|
-
|
|
23
|
+
@dataclass
|
|
24
|
+
class SerializedJudgeDecision:
|
|
23
25
|
"""Serialized judge decision for storage in ensemble metadata."""
|
|
24
26
|
|
|
25
27
|
winner_id: str
|
|
@@ -32,10 +34,23 @@ class SerializedJudgeDecision(TypedDict):
|
|
|
32
34
|
generation_time_ms: int | None
|
|
33
35
|
|
|
34
36
|
|
|
37
|
+
@dataclass
|
|
38
|
+
class EnsembleResult:
|
|
39
|
+
"""Result from ensemble judging combining multiple judge decisions."""
|
|
40
|
+
|
|
41
|
+
final_winner_id: str
|
|
42
|
+
final_margin: float
|
|
43
|
+
ensemble_method: str
|
|
44
|
+
num_judges: int
|
|
45
|
+
consensus_level: float
|
|
46
|
+
summary_reasoning: str
|
|
47
|
+
summary_feedback: str
|
|
48
|
+
|
|
49
|
+
|
|
35
50
|
def calculate_ensemble_result(
|
|
36
51
|
decisions: list[JudgeDecision],
|
|
37
52
|
context: DebateContext,
|
|
38
|
-
) ->
|
|
53
|
+
) -> EnsembleResult:
|
|
39
54
|
"""Calculate ensemble result from multiple judge decisions."""
|
|
40
55
|
|
|
41
56
|
# Debug: Check if all decisions are identical
|
|
@@ -146,41 +161,41 @@ def calculate_ensemble_result(
|
|
|
146
161
|
# Calculate consensus level (agreement rate)
|
|
147
162
|
consensus_level = max_votes / len(decisions) if decisions else 0.0
|
|
148
163
|
|
|
149
|
-
return
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
164
|
+
return EnsembleResult(
|
|
165
|
+
final_winner_id=ensemble_winner,
|
|
166
|
+
final_margin=ensemble_margin,
|
|
167
|
+
ensemble_method="majority_vote_with_tiebreaker",
|
|
168
|
+
num_judges=len(decisions),
|
|
169
|
+
consensus_level=consensus_level,
|
|
170
|
+
summary_reasoning=f"Winner determined by {decision_method}",
|
|
171
|
+
summary_feedback=(
|
|
157
172
|
f"Ensemble decision from {len(decisions)} judges. Consensus level:"
|
|
158
173
|
f" {consensus_level:.1%}"
|
|
159
174
|
),
|
|
160
|
-
|
|
175
|
+
)
|
|
161
176
|
|
|
162
177
|
|
|
163
178
|
def serialize_individual_decision(decision: JudgeDecision) -> SerializedJudgeDecision:
|
|
164
179
|
"""Serialize an individual judge decision for storage in ensemble metadata."""
|
|
165
|
-
return
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
180
|
+
return SerializedJudgeDecision(
|
|
181
|
+
winner_id=decision.winner_id,
|
|
182
|
+
winner_margin=decision.winner_margin,
|
|
183
|
+
overall_feedback=decision.overall_feedback,
|
|
184
|
+
reasoning=decision.reasoning,
|
|
185
|
+
criterion_scores=[
|
|
186
|
+
SerializedCriterionScore(
|
|
187
|
+
criterion=(
|
|
173
188
|
score.criterion
|
|
174
189
|
if isinstance(score.criterion, str)
|
|
175
190
|
else score.criterion.value
|
|
176
191
|
),
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
192
|
+
participant_id=score.participant_id,
|
|
193
|
+
score=score.score,
|
|
194
|
+
feedback=score.feedback,
|
|
195
|
+
)
|
|
181
196
|
for score in decision.criterion_scores
|
|
182
197
|
],
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
198
|
+
judge_model=decision.judge_model,
|
|
199
|
+
judge_provider=decision.judge_provider,
|
|
200
|
+
generation_time_ms=decision.generation_time_ms,
|
|
201
|
+
)
|