dialectus-engine 0.3.0__tar.gz → 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. {dialectus_engine-0.3.0/dialectus_engine.egg-info → dialectus_engine-0.3.1}/PKG-INFO +1 -1
  2. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/__init__.py +1 -1
  3. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/config/settings.py +41 -16
  4. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/context_builder.py +11 -7
  5. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/engine.py +8 -11
  6. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/round_manager.py +23 -23
  7. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/types.py +8 -6
  8. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/utils.py +11 -6
  9. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/judges/ai_judge.py +20 -9
  10. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/judges/base.py +32 -43
  11. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/judges/ensemble_utils.py +44 -29
  12. dialectus_engine-0.3.1/dialectus/engine/models/__init__.py +17 -0
  13. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/manager.py +12 -8
  14. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_filter_config.py +3 -1
  15. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/providers/base_model_provider.py +31 -9
  16. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/providers/ollama_provider.py +7 -7
  17. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/providers/open_router_provider.py +93 -53
  18. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/providers/openrouter_generation_types.py +44 -3
  19. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/moderation/openai_moderator.py +20 -55
  20. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1/dialectus_engine.egg-info}/PKG-INFO +1 -1
  21. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/pyproject.toml +2 -1
  22. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/tests/test_moderation_openai.py +39 -6
  23. dialectus_engine-0.3.0/dialectus/engine/models/__init__.py +0 -1
  24. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/LICENSE +0 -0
  25. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/README.md +0 -0
  26. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/__init__.py +0 -0
  27. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/config/__init__.py +0 -0
  28. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/config/openrouter_filters.json +0 -0
  29. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/__init__.py +0 -0
  30. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/judge_coordinator.py +0 -0
  31. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/models.py +0 -0
  32. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/prompt_builder.py +0 -0
  33. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/debate_engine/response_handler.py +0 -0
  34. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/formats/__init__.py +0 -0
  35. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/formats/base.py +0 -0
  36. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/formats/oxford.py +0 -0
  37. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/formats/parliamentary.py +0 -0
  38. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/formats/public_forum.py +0 -0
  39. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/formats/registry.py +0 -0
  40. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/formats/socratic.py +0 -0
  41. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/formats/topic_themes.py +0 -0
  42. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/judges/__init__.py +0 -0
  43. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/judges/factory.py +0 -0
  44. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/base_types.py +0 -0
  45. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/cache_manager.py +0 -0
  46. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_architecture.py +0 -0
  47. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_capability_extractor.py +0 -0
  48. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_enhanced_model_info.py +0 -0
  49. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_model.py +0 -0
  50. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_model_filter.py +0 -0
  51. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_models_response.py +0 -0
  52. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_pricing.py +0 -0
  53. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/openrouter/openrouter_top_provider.py +0 -0
  54. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/providers/__init__.py +0 -0
  55. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/providers/exceptions.py +0 -0
  56. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/models/providers/providers.py +0 -0
  57. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/moderation/__init__.py +0 -0
  58. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/moderation/base_moderator.py +0 -0
  59. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/moderation/exceptions.py +0 -0
  60. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/moderation/llm_moderator.py +0 -0
  61. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus/engine/moderation/manager.py +0 -0
  62. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus_engine.egg-info/SOURCES.txt +0 -0
  63. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus_engine.egg-info/dependency_links.txt +0 -0
  64. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus_engine.egg-info/requires.txt +0 -0
  65. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/dialectus_engine.egg-info/top_level.txt +0 -0
  66. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/setup.cfg +0 -0
  67. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/tests/test_cache_manager.py +0 -0
  68. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/tests/test_config_settings.py +0 -0
  69. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/tests/test_debate_models.py +0 -0
  70. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/tests/test_format_registry.py +0 -0
  71. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/tests/test_formats_base.py +0 -0
  72. {dialectus_engine-0.3.0 → dialectus_engine-0.3.1}/tests/test_judges_factory.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dialectus-engine
3
- Version: 0.3.0
3
+ Version: 0.3.1
4
4
  Summary: Dialectus AI debate engine framework
5
5
  Author: Dialectus AI
6
6
  License-Expression: MIT
@@ -4,4 +4,4 @@ from pkgutil import extend_path
4
4
 
5
5
  __path__ = extend_path(__path__, __name__)
6
6
  __all__: list[str] = []
7
- __version__ = "0.3.0"
7
+ __version__ = "0.3.1"
@@ -6,6 +6,25 @@ from typing import Literal
6
6
  from pydantic import BaseModel, Field, field_validator
7
7
 
8
8
 
9
+ def validate_provider_value(value: str, valid_providers: set[str]) -> str:
10
+ """Validate provider string against allowed values.
11
+
12
+ Args:
13
+ value: Provider name to validate
14
+ valid_providers: Set of valid provider names
15
+
16
+ Returns:
17
+ Lowercase provider name
18
+
19
+ Raises:
20
+ ValueError: If provider is not in valid_providers
21
+ """
22
+ normalized = value.lower()
23
+ if normalized not in valid_providers:
24
+ raise ValueError(f"Provider must be one of: {valid_providers}")
25
+ return normalized
26
+
27
+
9
28
  class ModelConfig(BaseModel):
10
29
  """Configuration for a debate model."""
11
30
 
@@ -24,11 +43,8 @@ class ModelConfig(BaseModel):
24
43
 
25
44
  @field_validator("provider")
26
45
  @classmethod
27
- def validate_provider(cls, v: str):
28
- valid_providers = {"ollama", "openrouter"}
29
- if v not in valid_providers:
30
- raise ValueError(f"Provider must be one of: {valid_providers}")
31
- return v
46
+ def validate_provider(cls, v: str) -> str:
47
+ return validate_provider_value(v, {"ollama", "openrouter"})
32
48
 
33
49
 
34
50
  class DebateConfig(BaseModel):
@@ -93,9 +109,9 @@ class ModerationConfig(BaseModel):
93
109
 
94
110
  @field_validator("provider")
95
111
  @classmethod
96
- def validate_provider(cls, v: str):
112
+ def validate_provider(cls, v: str) -> str:
97
113
  # Provider names are case-insensitive
98
- # Ollama and openrouter have automatic URL/key resolution
114
+ # For moderation, we accept any provider (ollama, openrouter, openai, custom)
99
115
  # Custom providers require explicit base_url
100
116
  return v.lower()
101
117
 
@@ -183,11 +199,8 @@ class SystemConfig(BaseModel):
183
199
 
184
200
  @field_validator("debate_topic_source")
185
201
  @classmethod
186
- def validate_topic_source(cls, v: str):
187
- valid_sources = {"ollama", "openrouter"}
188
- if v not in valid_sources:
189
- raise ValueError(f"debate_topic_source must be one of: {valid_sources}")
190
- return v
202
+ def validate_topic_source(cls, v: str) -> str:
203
+ return validate_provider_value(v, {"ollama", "openrouter"})
191
204
 
192
205
 
193
206
  class AppConfig(BaseModel):
@@ -203,14 +216,26 @@ class AppConfig(BaseModel):
203
216
 
204
217
  @classmethod
205
218
  def load_from_file(cls, config_path: Path) -> "AppConfig":
206
- """Load configuration from JSON file."""
219
+ """Load configuration from JSON file.
220
+
221
+ Args:
222
+ config_path: Path to the JSON configuration file
223
+
224
+ Returns:
225
+ AppConfig instance loaded from the file
226
+
227
+ Raises:
228
+ FileNotFoundError: If config file doesn't exist
229
+ ValueError: If required sections are missing or invalid
230
+ """
207
231
  if not config_path.exists():
208
232
  raise FileNotFoundError(f"Config file not found: {config_path}")
209
233
 
234
+ # Use Pydantic's built-in JSON validation
210
235
  import json
211
236
 
212
- with open(config_path, encoding="utf-8") as f:
213
- data = json.load(f)
237
+ json_data = config_path.read_text(encoding="utf-8")
238
+ data = json.loads(json_data)
214
239
 
215
240
  # Validate required sections
216
241
  required_sections = ["debate", "models", "judging", "system"]
@@ -226,7 +251,7 @@ class AppConfig(BaseModel):
226
251
  "Config must include at least one model in 'models' section"
227
252
  )
228
253
 
229
- return cls(**data)
254
+ return cls.model_validate(data)
230
255
 
231
256
 
232
257
  def get_default_config() -> AppConfig:
@@ -2,9 +2,10 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
- from typing import TYPE_CHECKING
5
+ from typing import TYPE_CHECKING, cast
6
6
 
7
7
  from dialectus.engine.config.settings import AppConfig
8
+ from dialectus.engine.models.providers.base_model_provider import ChatMessage
8
9
 
9
10
  from .models import DebateContext
10
11
  from .prompt_builder import PromptBuilder
@@ -13,6 +14,9 @@ from .types import DebatePhase
13
14
  if TYPE_CHECKING:
14
15
  from dialectus.engine.formats import DebateFormat, FormatPhase
15
16
 
17
+ # Maximum number of recent messages to include in conversation context
18
+ MAX_CONTEXT_MESSAGES = 6
19
+
16
20
 
17
21
  class ContextBuilder:
18
22
  """Builds conversation context for model prompts during debates."""
@@ -39,7 +43,7 @@ class ContextBuilder:
39
43
  speaker_id: str,
40
44
  phase: DebatePhase,
41
45
  context: DebateContext,
42
- ) -> list[dict[str, str]]:
46
+ ) -> list[ChatMessage]:
43
47
  """Build conversation context for the model.
44
48
 
45
49
  Args:
@@ -71,7 +75,7 @@ class ContextBuilder:
71
75
  })
72
76
 
73
77
  # Add relevant debate history
74
- for msg in context.messages[-6:]: # Last 6 messages for context
78
+ for msg in context.messages[-MAX_CONTEXT_MESSAGES:]:
75
79
  role = "assistant" if msg.speaker_id == speaker_id else "user"
76
80
 
77
81
  # Use a cleaner format that's less likely to be repeated by small
@@ -94,14 +98,14 @@ class ContextBuilder:
94
98
  )
95
99
  messages.append({"role": "user", "content": turn_prompt})
96
100
 
97
- return messages
101
+ return cast(list[ChatMessage], messages)
98
102
 
99
103
  def build_format_conversation_context(
100
104
  self,
101
105
  speaker_id: str,
102
106
  format_phase: FormatPhase,
103
107
  context: DebateContext,
104
- ) -> list[dict[str, str]]:
108
+ ) -> list[ChatMessage]:
105
109
  """Build conversation context for the model using format phase.
106
110
 
107
111
  Args:
@@ -139,7 +143,7 @@ class ContextBuilder:
139
143
  })
140
144
 
141
145
  # Add relevant debate history
142
- for msg in context.messages[-6:]: # Last 6 messages for context
146
+ for msg in context.messages[-MAX_CONTEXT_MESSAGES:]:
143
147
  role = "assistant" if msg.speaker_id == speaker_id else "user"
144
148
 
145
149
  # Use a cleaner format that's less likely to be repeated by small
@@ -160,4 +164,4 @@ class ContextBuilder:
160
164
  turn_prompt = f"Now speak as {role_name}. Stay under {word_limit} words."
161
165
  messages.append({"role": "user", "content": turn_prompt})
162
166
 
163
- return messages
167
+ return cast(list[ChatMessage], messages)
@@ -22,6 +22,7 @@ from .types import (
22
22
  MessageEventCallback,
23
23
  PhaseEventCallback,
24
24
  PhaseEventType,
25
+ PhaseStartedEventData,
25
26
  )
26
27
 
27
28
  logger = logging.getLogger(__name__)
@@ -169,18 +170,14 @@ class DebateEngine:
169
170
 
170
171
  # Invoke phase callback if provided
171
172
  if phase_callback:
172
- await phase_callback(
173
- PhaseEventType.PHASE_STARTED,
174
- {
175
- "phase": format_phase.name,
176
- "instruction": format_phase.instruction,
177
- "current_phase": self.context.current_round,
178
- "total_phases": total_phases,
179
- "progress_percentage": round(
180
- (phase_index / total_phases) * 100
181
- ),
182
- },
173
+ phase_event = PhaseStartedEventData(
174
+ phase=format_phase.name,
175
+ instruction=format_phase.instruction,
176
+ current_phase=self.context.current_round,
177
+ total_phases=total_phases,
178
+ progress_percentage=round((phase_index / total_phases) * 100),
183
179
  )
180
+ await phase_callback(PhaseEventType.PHASE_STARTED, phase_event)
184
181
 
185
182
  # If streaming callbacks provided, use streaming version
186
183
  if message_callback or chunk_callback:
@@ -11,8 +11,10 @@ from .response_handler import ResponseHandler
11
11
  from .types import (
12
12
  ChunkCallback,
13
13
  DebatePhase,
14
+ MessageCompleteEventData,
14
15
  MessageEventCallback,
15
16
  MessageEventType,
17
+ MessageStartEventData,
16
18
  Position,
17
19
  )
18
20
 
@@ -129,16 +131,14 @@ class RoundManager:
129
131
 
130
132
  # Call message_start callback
131
133
  if message_callback:
132
- await message_callback(
133
- MessageEventType.MESSAGE_START,
134
- {
135
- "message_id": message_id,
136
- "speaker_id": speaker_id,
137
- "position": speaker_position.value,
138
- "phase": format_phase.phase.value,
139
- "round_number": context.current_round,
140
- },
134
+ start_event = MessageStartEventData(
135
+ message_id=message_id,
136
+ speaker_id=speaker_id,
137
+ position=speaker_position.value,
138
+ phase=format_phase.phase.value,
139
+ round_number=context.current_round,
141
140
  )
141
+ await message_callback(MessageEventType.MESSAGE_START, start_event)
142
142
 
143
143
  # Create streaming wrapper that calls chunk_callback
144
144
  async def chunk_wrapper(chunk: str, is_complete: bool):
@@ -154,21 +154,21 @@ class RoundManager:
154
154
 
155
155
  # Call message_complete callback
156
156
  if message_callback:
157
+ complete_event = MessageCompleteEventData(
158
+ message_id=message_id,
159
+ speaker_id=message.speaker_id,
160
+ position=message.position.value,
161
+ phase=message.phase.value,
162
+ round_number=message.round_number,
163
+ content=message.content,
164
+ timestamp=message.timestamp.isoformat(),
165
+ word_count=len(message.content.split()),
166
+ metadata=message.metadata,
167
+ cost=message.cost,
168
+ generation_id=message.generation_id,
169
+ )
157
170
  await message_callback(
158
- MessageEventType.MESSAGE_COMPLETE,
159
- {
160
- "message_id": message_id,
161
- "speaker_id": message.speaker_id,
162
- "position": message.position.value,
163
- "phase": message.phase.value,
164
- "round_number": message.round_number,
165
- "content": message.content,
166
- "timestamp": message.timestamp.isoformat(),
167
- "word_count": len(message.content.split()),
168
- "metadata": message.metadata,
169
- "cost": message.cost,
170
- "generation_id": message.generation_id,
171
- },
171
+ MessageEventType.MESSAGE_COMPLETE, complete_event
172
172
  )
173
173
 
174
174
  logger.info(
@@ -2,7 +2,9 @@
2
2
 
3
3
  from collections.abc import Awaitable, Callable
4
4
  from enum import Enum
5
- from typing import Protocol, TypedDict
5
+ from typing import Protocol
6
+
7
+ from pydantic import BaseModel
6
8
 
7
9
 
8
10
  class PhaseEventType(str, Enum):
@@ -19,7 +21,7 @@ class MessageEventType(str, Enum):
19
21
  MESSAGE_COMPLETE = "message_complete"
20
22
 
21
23
 
22
- class PhaseStartedEventData(TypedDict):
24
+ class PhaseStartedEventData(BaseModel):
23
25
  """Data structure for phase_started event callbacks."""
24
26
 
25
27
  phase: str
@@ -29,7 +31,7 @@ class PhaseStartedEventData(TypedDict):
29
31
  progress_percentage: int
30
32
 
31
33
 
32
- class MessageStartEventData(TypedDict):
34
+ class MessageStartEventData(BaseModel):
33
35
  """Data structure for message_start event callbacks."""
34
36
 
35
37
  message_id: str
@@ -39,7 +41,7 @@ class MessageStartEventData(TypedDict):
39
41
  round_number: int
40
42
 
41
43
 
42
- class MessageCompleteEventData(TypedDict):
44
+ class MessageCompleteEventData(BaseModel):
43
45
  """Data structure for message_complete event callbacks."""
44
46
 
45
47
  message_id: str
@@ -51,8 +53,8 @@ class MessageCompleteEventData(TypedDict):
51
53
  timestamp: str
52
54
  word_count: int
53
55
  metadata: dict[str, str]
54
- cost: float | None
55
- generation_id: str | None
56
+ cost: float | None = None
57
+ generation_id: str | None = None
56
58
 
57
59
 
58
60
  # Callback protocols with proper type narrowing
@@ -10,6 +10,14 @@ from .models import DebateMessage
10
10
 
11
11
  logger = logging.getLogger(__name__)
12
12
 
13
+ # Token calculation constants
14
+ # Average tokens per word ratio (1 token ≈ 0.75 words, so multiply words by 1.33)
15
+ TOKENS_PER_WORD_RATIO = 1.33
16
+ # Buffer for sentence completion in words
17
+ SENTENCE_BUFFER_WORDS = 100
18
+ # Buffer for sentence completion in tokens (100 words * 1.33)
19
+ SENTENCE_BUFFER_TOKENS = 133
20
+
13
21
 
14
22
  def calculate_max_tokens(word_limit: int) -> int:
15
23
  """Calculate maximum tokens based on word limit with buffer for incomplete
@@ -19,13 +27,10 @@ def calculate_max_tokens(word_limit: int) -> int:
19
27
  word_limit: Maximum number of words allowed
20
28
 
21
29
  Returns:
22
- Maximum number of tokens with ~100 word buffer (1 token ≈ 0.75 words)
30
+ Maximum number of tokens with buffer for sentence completion
23
31
  """
24
- # Base conversion: word_limit * 1.33 (since 1 token ≈ 0.75 words)
25
- # Add buffer: ~100 words = 133 tokens
26
- base_tokens = int(word_limit * 1.33)
27
- buffer_tokens = 133 # ~100 words of buffer for sentence completion
28
- return base_tokens + buffer_tokens
32
+ base_tokens = int(word_limit * TOKENS_PER_WORD_RATIO)
33
+ return base_tokens + SENTENCE_BUFFER_TOKENS
29
34
 
30
35
 
31
36
  def trim_incomplete_sentence(text: str) -> str:
@@ -8,10 +8,12 @@ import time
8
8
  import uuid
9
9
  from datetime import datetime
10
10
  from textwrap import dedent
11
+ from typing import cast
11
12
 
12
13
  from dialectus.engine.config.settings import SystemConfig
13
14
  from dialectus.engine.debate_engine.models import DebateContext
14
15
  from dialectus.engine.models.manager import ModelManager
16
+ from dialectus.engine.models.providers.base_model_provider import ChatMessage
15
17
 
16
18
  from .base import BaseJudge, CriterionScore, JudgeDecision, JudgmentCriterion
17
19
 
@@ -38,6 +40,12 @@ class AIJudge(BaseJudge):
38
40
  # Initialize ensemble temperature (can be overridden for ensemble judges)
39
41
  self._ensemble_temperature: float = 0.3
40
42
 
43
+ # Generation metadata - tracked per evaluation
44
+ self._last_generation_time_ms: int | None = None
45
+ self._last_generation_cost: float | None = None
46
+ self._last_generation_id: str | None = None
47
+ self._last_generation_provider: str | None = None
48
+
41
49
  # Generate unique judge ID for model manager registration
42
50
  self.judge_id = str(uuid.uuid4())
43
51
 
@@ -166,17 +174,20 @@ class AIJudge(BaseJudge):
166
174
  transcript, participants, criteria_list, context
167
175
  )
168
176
 
169
- messages = [
170
- {"role": "system", "content": self._get_judge_system_prompt()},
171
- {"role": "user", "content": evaluation_prompt},
172
- ]
177
+ messages: list[ChatMessage] = cast(
178
+ list[ChatMessage],
179
+ [
180
+ {"role": "system", "content": self._get_judge_system_prompt()},
181
+ {"role": "user", "content": evaluation_prompt},
182
+ ],
183
+ )
173
184
 
174
185
  # Capture evaluation timing
175
186
  start_time = time.time()
176
187
 
177
188
  # Use model manager to generate response
178
189
  # Add some randomness for ensemble judges to get different evaluations
179
- temperature = getattr(self, "_ensemble_temperature", 0.3)
190
+ temperature = self._ensemble_temperature
180
191
 
181
192
  async with self.model_manager.model_session(self.judge_id):
182
193
  generation_metadata = (
@@ -459,15 +470,15 @@ parsed programmatically. Be thorough but concise in your reasoning."""
459
470
  reasoning=reasoning,
460
471
  judge_model=self.judge_model_name,
461
472
  judge_provider=self.judge_provider,
462
- generation_time_ms=getattr(self, "_last_generation_time_ms", None),
463
- cost=getattr(self, "_last_generation_cost", None),
464
- generation_id=getattr(self, "_last_generation_id", None),
473
+ generation_time_ms=self._last_generation_time_ms,
474
+ cost=self._last_generation_cost,
475
+ generation_id=self._last_generation_id,
465
476
  )
466
477
 
467
478
  # Schedule background cost query for OpenRouter judges
468
479
  if (
469
480
  judge_decision.generation_id
470
- and getattr(self, "_last_generation_provider", None) == "openrouter"
481
+ and self._last_generation_provider == "openrouter"
471
482
  ):
472
483
  asyncio.create_task(self._query_and_update_judge_cost(judge_decision))
473
484
 
@@ -62,6 +62,35 @@ class BaseJudge(ABC):
62
62
  """Judge name/identifier."""
63
63
  pass
64
64
 
65
+ def _get_participant_averages(
66
+ self, criterion_scores: list[CriterionScore]
67
+ ) -> dict[str, float]:
68
+ """Calculate average scores for each participant across all criteria.
69
+
70
+ Args:
71
+ criterion_scores: List of criterion scores from the debate
72
+
73
+ Returns:
74
+ Dictionary mapping participant IDs to their average scores
75
+ """
76
+ participant_totals: dict[str, float] = {}
77
+ participant_counts: dict[str, int] = {}
78
+
79
+ for score in criterion_scores:
80
+ participant_id = score.participant_id
81
+ participant_totals[participant_id] = (
82
+ participant_totals.get(participant_id, 0.0) + score.score
83
+ )
84
+ participant_counts[participant_id] = (
85
+ participant_counts.get(participant_id, 0) + 1
86
+ )
87
+
88
+ return {
89
+ pid: participant_totals[pid] / participant_counts[pid]
90
+ for pid in participant_totals
91
+ if participant_counts[pid] > 0
92
+ }
93
+
65
94
  def _calculate_total_score(
66
95
  self, scores: list[CriterionScore], participant_id: str
67
96
  ) -> float:
@@ -80,34 +109,13 @@ class BaseJudge(ABC):
80
109
  if not criterion_scores:
81
110
  return 0.0
82
111
 
83
- # Group scores by participant
84
- participant_totals: dict[str, float] = {}
85
- participant_counts: dict[str, int] = {}
86
-
87
- for score in criterion_scores:
88
- participant_id = score.participant_id
89
- if participant_id not in participant_totals:
90
- participant_totals[participant_id] = 0.0
91
- participant_counts[participant_id] = 0
92
-
93
- participant_totals[participant_id] += score.score
94
- participant_counts[participant_id] += 1
95
-
96
- # Calculate average scores for each participant
97
- participant_averages: dict[str, float] = {}
98
- for participant_id in participant_totals:
99
- if participant_counts[participant_id] > 0:
100
- participant_averages[participant_id] = (
101
- participant_totals[participant_id]
102
- / participant_counts[participant_id]
103
- )
112
+ participant_averages = self._get_participant_averages(criterion_scores)
104
113
 
105
114
  # Find margin between highest and second highest
106
115
  if len(participant_averages) < 2:
107
116
  return 0.0
108
117
 
109
- averages = list(participant_averages.values())
110
- averages.sort(reverse=True)
118
+ averages = sorted(participant_averages.values(), reverse=True)
111
119
  return averages[0] - averages[1]
112
120
 
113
121
  def _determine_winner_from_scores(
@@ -117,26 +125,7 @@ class BaseJudge(ABC):
117
125
  if not criterion_scores:
118
126
  return "unknown"
119
127
 
120
- participant_averages: dict[str, float] = {}
121
- participant_totals: dict[str, float] = {}
122
- participant_counts: dict[str, int] = {}
123
-
124
- for score in criterion_scores:
125
- participant_id = score.participant_id
126
- if participant_id not in participant_totals:
127
- participant_totals[participant_id] = 0.0
128
- participant_counts[participant_id] = 0
129
-
130
- participant_totals[participant_id] += score.score
131
- participant_counts[participant_id] += 1
132
-
133
- # Calculate averages
134
- for participant_id in participant_totals:
135
- if participant_counts[participant_id] > 0:
136
- participant_averages[participant_id] = (
137
- participant_totals[participant_id]
138
- / participant_counts[participant_id]
139
- )
128
+ participant_averages = self._get_participant_averages(criterion_scores)
140
129
 
141
130
  # Return participant with highest average
142
131
  return (
@@ -1,7 +1,7 @@
1
1
  """Ensemble judging utilities - standalone functions for combining judge decisions."""
2
2
 
3
3
  import logging
4
- from typing import Any, TypedDict
4
+ from dataclasses import dataclass
5
5
 
6
6
  from dialectus.engine.debate_engine.models import DebateContext
7
7
 
@@ -10,7 +10,8 @@ from .base import CriterionScore, JudgeDecision
10
10
  logger = logging.getLogger(__name__)
11
11
 
12
12
 
13
- class SerializedCriterionScore(TypedDict):
13
+ @dataclass
14
+ class SerializedCriterionScore:
14
15
  """Serialized criterion score for storage."""
15
16
 
16
17
  criterion: str
@@ -19,7 +20,8 @@ class SerializedCriterionScore(TypedDict):
19
20
  feedback: str
20
21
 
21
22
 
22
- class SerializedJudgeDecision(TypedDict):
23
+ @dataclass
24
+ class SerializedJudgeDecision:
23
25
  """Serialized judge decision for storage in ensemble metadata."""
24
26
 
25
27
  winner_id: str
@@ -32,10 +34,23 @@ class SerializedJudgeDecision(TypedDict):
32
34
  generation_time_ms: int | None
33
35
 
34
36
 
37
+ @dataclass
38
+ class EnsembleResult:
39
+ """Result from ensemble judging combining multiple judge decisions."""
40
+
41
+ final_winner_id: str
42
+ final_margin: float
43
+ ensemble_method: str
44
+ num_judges: int
45
+ consensus_level: float
46
+ summary_reasoning: str
47
+ summary_feedback: str
48
+
49
+
35
50
  def calculate_ensemble_result(
36
51
  decisions: list[JudgeDecision],
37
52
  context: DebateContext,
38
- ) -> dict[str, Any]:
53
+ ) -> EnsembleResult:
39
54
  """Calculate ensemble result from multiple judge decisions."""
40
55
 
41
56
  # Debug: Check if all decisions are identical
@@ -146,41 +161,41 @@ def calculate_ensemble_result(
146
161
  # Calculate consensus level (agreement rate)
147
162
  consensus_level = max_votes / len(decisions) if decisions else 0.0
148
163
 
149
- return {
150
- "final_winner_id": ensemble_winner,
151
- "final_margin": ensemble_margin,
152
- "ensemble_method": "majority_vote_with_tiebreaker",
153
- "num_judges": len(decisions),
154
- "consensus_level": consensus_level,
155
- "summary_reasoning": f"Winner determined by {decision_method}",
156
- "summary_feedback": (
164
+ return EnsembleResult(
165
+ final_winner_id=ensemble_winner,
166
+ final_margin=ensemble_margin,
167
+ ensemble_method="majority_vote_with_tiebreaker",
168
+ num_judges=len(decisions),
169
+ consensus_level=consensus_level,
170
+ summary_reasoning=f"Winner determined by {decision_method}",
171
+ summary_feedback=(
157
172
  f"Ensemble decision from {len(decisions)} judges. Consensus level:"
158
173
  f" {consensus_level:.1%}"
159
174
  ),
160
- }
175
+ )
161
176
 
162
177
 
163
178
  def serialize_individual_decision(decision: JudgeDecision) -> SerializedJudgeDecision:
164
179
  """Serialize an individual judge decision for storage in ensemble metadata."""
165
- return {
166
- "winner_id": decision.winner_id,
167
- "winner_margin": decision.winner_margin,
168
- "overall_feedback": decision.overall_feedback,
169
- "reasoning": decision.reasoning,
170
- "criterion_scores": [
171
- {
172
- "criterion": (
180
+ return SerializedJudgeDecision(
181
+ winner_id=decision.winner_id,
182
+ winner_margin=decision.winner_margin,
183
+ overall_feedback=decision.overall_feedback,
184
+ reasoning=decision.reasoning,
185
+ criterion_scores=[
186
+ SerializedCriterionScore(
187
+ criterion=(
173
188
  score.criterion
174
189
  if isinstance(score.criterion, str)
175
190
  else score.criterion.value
176
191
  ),
177
- "participant_id": score.participant_id,
178
- "score": score.score,
179
- "feedback": score.feedback,
180
- }
192
+ participant_id=score.participant_id,
193
+ score=score.score,
194
+ feedback=score.feedback,
195
+ )
181
196
  for score in decision.criterion_scores
182
197
  ],
183
- "judge_model": decision.judge_model,
184
- "judge_provider": decision.judge_provider,
185
- "generation_time_ms": decision.generation_time_ms,
186
- }
198
+ judge_model=decision.judge_model,
199
+ judge_provider=decision.judge_provider,
200
+ generation_time_ms=decision.generation_time_ms,
201
+ )