classone 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
classone/sdk.py ADDED
@@ -0,0 +1,290 @@
1
+ """Python Client SDK for ClassOne System 1 decision models.
2
+
3
+ Provides:
4
+ - Noul, Choice, Score question builder primitives
5
+ - Synchronous ClassOneClient
6
+ - Asynchronous AsyncClassOneClient
7
+ - Typed ClassOneResponse with grouped accessors (.nouls, .choices, .scores)
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import os
13
+ from typing import Any, Union
14
+
15
+ import httpx
16
+ from pydantic import BaseModel, Field
17
+
18
+ from classone.schemas import DecisionUsage
19
+
20
+
21
+ class Noul:
22
+ """Boolean yes/no question builder primitive."""
23
+
24
+ def __init__(self, instructions: str):
25
+ self.instructions = instructions
26
+
27
+ def to_dict(self) -> dict[str, Any]:
28
+ return {"type": "noul", "instructions": self.instructions}
29
+
30
+
31
+ class Choice:
32
+ """Categorical classification question builder primitive."""
33
+
34
+ def __init__(self, instructions: str, criteria: dict[str, str]):
35
+ self.instructions = instructions
36
+ self.criteria = criteria
37
+
38
+ if len(criteria) < 2:
39
+ raise ValueError("Choice question must have at least 2 options in criteria.")
40
+ if len(criteria) > 255:
41
+ raise ValueError("Choice question cannot exceed 255 options.")
42
+
43
+ def to_dict(self) -> dict[str, Any]:
44
+ return {
45
+ "type": "choice",
46
+ "instructions": self.instructions,
47
+ "criteria": self.criteria,
48
+ }
49
+
50
+
51
+ class Score:
52
+ """Ordinal rubric question builder primitive."""
53
+
54
+ def __init__(self, instructions: str, criteria: list[str]):
55
+ self.instructions = instructions
56
+ self.criteria = criteria
57
+
58
+ if len(criteria) < 2:
59
+ raise ValueError("Score question must have at least 2 rubric levels.")
60
+ if len(criteria) > 10:
61
+ raise ValueError("Score question cannot exceed 10 rubric levels.")
62
+
63
+ def to_dict(self) -> dict[str, Any]:
64
+ return {
65
+ "type": "score",
66
+ "instructions": self.instructions,
67
+ "criteria": self.criteria,
68
+ }
69
+
70
+
71
+ class NoulAnswer(BaseModel):
72
+ type: str = "noul"
73
+ noul: float = Field(..., ge=0.0, le=1.0)
74
+
75
+
76
+ class ChoiceAnswer(BaseModel):
77
+ type: str = "choice"
78
+ choice: str
79
+ probabilities: dict[str, float]
80
+ confidence: float = Field(..., ge=0.0, le=1.0)
81
+
82
+
83
+ class ScoreAnswer(BaseModel):
84
+ type: str = "score"
85
+ score: float
86
+ probabilities: dict[str, float]
87
+ confidence: float = Field(..., ge=0.0, le=1.0)
88
+
89
+
90
+ QuestionPrimitive = Union[Noul, Choice, Score]
91
+
92
+
93
+ class ClassOneResponse:
94
+ """Structured response container from a ClassOne decision request."""
95
+
96
+ def __init__(
97
+ self,
98
+ model: str,
99
+ answers: dict[str, Any],
100
+ usage: dict[str, int] | None = None,
101
+ ):
102
+ self.model = model
103
+ self.raw_answers = answers
104
+ self.usage = DecisionUsage(**(usage or {}))
105
+
106
+ self.nouls: dict[str, NoulAnswer] = {}
107
+ self.choices: dict[str, ChoiceAnswer] = {}
108
+ self.scores: dict[str, ScoreAnswer] = {}
109
+ self.answers: dict[str, NoulAnswer | ChoiceAnswer | ScoreAnswer] = {}
110
+
111
+ for q_id, item in answers.items():
112
+ if hasattr(item, "model_dump"):
113
+ val = item.model_dump()
114
+ elif isinstance(item, dict):
115
+ val = item
116
+ else:
117
+ val = {"type": getattr(item, "type", None)}
118
+
119
+ ans_type = val.get("type")
120
+ if ans_type == "noul":
121
+ noul_ans = NoulAnswer(noul=val["noul"])
122
+ self.nouls[q_id] = noul_ans
123
+ self.answers[q_id] = noul_ans
124
+ elif ans_type == "choice":
125
+ choice_ans = ChoiceAnswer(
126
+ choice=val["choice"],
127
+ probabilities=val["probabilities"],
128
+ confidence=val["confidence"],
129
+ )
130
+ self.choices[q_id] = choice_ans
131
+ self.answers[q_id] = choice_ans
132
+ elif ans_type == "score":
133
+ score_ans = ScoreAnswer(
134
+ score=val["score"],
135
+ probabilities=val["probabilities"],
136
+ confidence=val["confidence"],
137
+ )
138
+ self.scores[q_id] = score_ans
139
+ self.answers[q_id] = score_ans
140
+
141
+ def __getitem__(self, key: str) -> NoulAnswer | ChoiceAnswer | ScoreAnswer:
142
+ return self.answers[key]
143
+
144
+ def __contains__(self, key: str) -> bool:
145
+ return key in self.answers
146
+
147
+
148
+ class BaseClassOneClient:
149
+ """Shared client configuration and payload formatting."""
150
+
151
+ def __init__(
152
+ self,
153
+ api_key: str | None = None,
154
+ base_url: str | None = None,
155
+ timeout: float = 30.0,
156
+ ):
157
+ self.api_key = api_key or os.environ.get("CLASSONE_API_KEY", "default-key")
158
+ self.base_url = (base_url or os.environ.get("CLASSONE_BASE_URL") or "http://localhost:8000").rstrip("/")
159
+ self.timeout = timeout
160
+
161
+ def _format_payload(
162
+ self,
163
+ state: str | dict[str, Any] | list[Any],
164
+ questions: dict[str, QuestionPrimitive],
165
+ model: str,
166
+ ) -> dict[str, Any]:
167
+ serialized_questions = {}
168
+ for q_id, q in questions.items():
169
+ if hasattr(q, "to_dict"):
170
+ serialized_questions[q_id] = q.to_dict()
171
+ elif isinstance(q, dict):
172
+ serialized_questions[q_id] = q
173
+ else:
174
+ raise TypeError(f"Invalid question object for '{q_id}': {type(q)}")
175
+
176
+ return {
177
+ "model": model,
178
+ "state": state,
179
+ "questions": serialized_questions,
180
+ }
181
+
182
+ def _headers(self) -> dict[str, str]:
183
+ headers = {"Content-Type": "application/json"}
184
+ if self.api_key:
185
+ headers["Authorization"] = f"Bearer {self.api_key}"
186
+ return headers
187
+
188
+
189
+ class ClassOneClient(BaseClassOneClient):
190
+ """Synchronous client for ClassOne decision models."""
191
+
192
+ def __init__(
193
+ self,
194
+ api_key: str | None = None,
195
+ base_url: str | None = None,
196
+ timeout: float = 30.0,
197
+ transport: httpx.BaseTransport | None = None,
198
+ http_client: Any | None = None,
199
+ ):
200
+ super().__init__(api_key=api_key, base_url=base_url, timeout=timeout)
201
+ if http_client is not None:
202
+ self._client = http_client
203
+ self._owns_client = False
204
+ else:
205
+ self._client = httpx.Client(
206
+ base_url=self.base_url,
207
+ headers=self._headers(),
208
+ timeout=self.timeout,
209
+ transport=transport,
210
+ )
211
+ self._owns_client = True
212
+
213
+ def __enter__(self) -> ClassOneClient:
214
+ return self
215
+
216
+ def __exit__(self, exc_type, exc_val, exc_tb):
217
+ self.close()
218
+
219
+ def close(self):
220
+ if getattr(self, "_owns_client", True):
221
+ self._client.close()
222
+
223
+ def decide(
224
+ self,
225
+ state: str | dict[str, Any] | list[Any],
226
+ questions: dict[str, QuestionPrimitive],
227
+ model: str = "class-one-gemma-4-e2b-it",
228
+ ) -> ClassOneResponse:
229
+ """Evaluates questions over state in a single synchronous call."""
230
+ payload = self._format_payload(state=state, questions=questions, model=model)
231
+ resp = self._client.post("/v1/decide", json=payload)
232
+ resp.raise_for_status()
233
+ data = resp.json()
234
+ return ClassOneResponse(
235
+ model=data.get("model", model),
236
+ answers=data.get("answers", {}),
237
+ usage=data.get("usage", {}),
238
+ )
239
+
240
+
241
+ class AsyncClassOneClient(BaseClassOneClient):
242
+ """Asynchronous client for ClassOne decision models."""
243
+
244
+ def __init__(
245
+ self,
246
+ api_key: str | None = None,
247
+ base_url: str | None = None,
248
+ timeout: float = 30.0,
249
+ transport: httpx.AsyncBaseTransport | None = None,
250
+ http_client: httpx.AsyncClient | None = None,
251
+ ):
252
+ super().__init__(api_key=api_key, base_url=base_url, timeout=timeout)
253
+ if http_client is not None:
254
+ self._client = http_client
255
+ self._owns_client = False
256
+ else:
257
+ self._client = httpx.AsyncClient(
258
+ base_url=self.base_url,
259
+ headers=self._headers(),
260
+ timeout=self.timeout,
261
+ transport=transport,
262
+ )
263
+ self._owns_client = True
264
+
265
+ async def __aenter__(self) -> AsyncClassOneClient:
266
+ return self
267
+
268
+ async def __aexit__(self, exc_type, exc_val, exc_tb):
269
+ await self.close()
270
+
271
+ async def close(self):
272
+ if getattr(self, "_owns_client", True):
273
+ await self._client.aclose()
274
+
275
+ async def decide(
276
+ self,
277
+ state: str | dict[str, Any] | list[Any],
278
+ questions: dict[str, QuestionPrimitive],
279
+ model: str = "class-one-gemma-4-e2b-it",
280
+ ) -> ClassOneResponse:
281
+ """Evaluates questions over state in an asynchronous call."""
282
+ payload = self._format_payload(state=state, questions=questions, model=model)
283
+ resp = await self._client.post("/v1/decide", json=payload)
284
+ resp.raise_for_status()
285
+ data = resp.json()
286
+ return ClassOneResponse(
287
+ model=data.get("model", model),
288
+ answers=data.get("answers", {}),
289
+ usage=data.get("usage", {}),
290
+ )
@@ -0,0 +1 @@
1
+ """API server implementation for ClassOne (/v1/decide)."""
classone/server/app.py ADDED
@@ -0,0 +1,131 @@
1
+ """FastAPI Server for ClassOne System 1 decision models.
2
+
3
+ Exposes the POST /v1/decide and POST /v1/classone endpoints.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import os
9
+ import threading
10
+ from contextlib import asynccontextmanager
11
+ from typing import Any
12
+
13
+ from fastapi import FastAPI, HTTPException, status
14
+ from transformers import AutoTokenizer
15
+
16
+ from classone.modeling.configuration_classone import ClassOneConfig
17
+ from classone.modeling.modeling_classone import ClassOneModel
18
+ from classone.schemas import DecisionRequest, DecisionResponse, DecisionUsage
19
+ from classone.tokenizer import ClassOnePromptBuilder
20
+
21
+ # Global thread-safe state
22
+ _pipeline_lock = threading.Lock()
23
+ _model: ClassOneModel | None = None
24
+ _tokenizer: Any | None = None
25
+ _prompt_builder: ClassOnePromptBuilder | None = None
26
+
27
+ MAX_QUESTIONS_PER_REQUEST = 128
28
+ MAX_TOKENS_PER_REQUEST = 8192
29
+
30
+
31
+ def get_pipeline() -> tuple[ClassOneModel, ClassOnePromptBuilder]:
32
+ """Lazily and thread-safely initializes or retrieves model and tokenizer pipeline."""
33
+ global _model, _tokenizer, _prompt_builder
34
+ if _model is None:
35
+ with _pipeline_lock:
36
+ if _model is None:
37
+ model_name = os.environ.get("CLASSONE_BASE_MODEL") or "standalone"
38
+ if model_name == "standalone":
39
+ # Lightweight in-memory standalone mode
40
+ from tokenizers import Tokenizer
41
+ from tokenizers.models import WordLevel
42
+ from tokenizers.pre_tokenizers import Whitespace
43
+ from transformers import PreTrainedTokenizerFast
44
+
45
+ vocab = {"[UNK]": 0, "[PAD]": 1, "[CLS]": 2, "[SEP]": 3, "[MASK]": 4}
46
+ tok = Tokenizer(WordLevel(vocab=vocab, unk_token="[UNK]"))
47
+ tok.pre_tokenizer = Whitespace()
48
+ _tokenizer = PreTrainedTokenizerFast(tokenizer_object=tok, unk_token="[UNK]", pad_token="[PAD]")
49
+ config = ClassOneConfig(hidden_size=256, head_hidden_size=128)
50
+ _model = ClassOneModel(config)
51
+ else:
52
+ _tokenizer = AutoTokenizer.from_pretrained(model_name)
53
+ _model = ClassOneModel.from_backbone(model_name, tokenizer=_tokenizer)
54
+
55
+ _prompt_builder = ClassOnePromptBuilder(_tokenizer)
56
+
57
+ return _model, _prompt_builder
58
+
59
+
60
+ def set_pipeline(model: ClassOneModel, prompt_builder: ClassOnePromptBuilder):
61
+ """Allows programmatic injection of model and tokenizer (e.g., during tests)."""
62
+ global _model, _prompt_builder
63
+ with _pipeline_lock:
64
+ _model = model
65
+ _prompt_builder = prompt_builder
66
+
67
+
68
+ @asynccontextmanager
69
+ async def lifespan(app: FastAPI):
70
+ """Lifecycle hook warming up model pipeline at server start."""
71
+ get_pipeline()
72
+ yield
73
+
74
+
75
+ app = FastAPI(
76
+ title="ClassOne System 1 API",
77
+ description="Open-source System 1 decision engine built on Gemma 4 E2B",
78
+ version="0.1.0",
79
+ lifespan=lifespan,
80
+ )
81
+
82
+
83
+ @app.get("/health")
84
+ def health_check():
85
+ return {"status": "ok", "service": "classone-system-1"}
86
+
87
+
88
+ @app.post("/v1/decide", response_model=DecisionResponse)
89
+ @app.post("/v1/classone", response_model=DecisionResponse)
90
+ def evaluate_decisions(request: DecisionRequest) -> DecisionResponse:
91
+ """Evaluates arbitrary typed questions against unstructured state in a single pass."""
92
+ # Input validation guards against DoS
93
+ if len(request.questions) > MAX_QUESTIONS_PER_REQUEST:
94
+ raise HTTPException(
95
+ status_code=status.HTTP_400_BAD_REQUEST,
96
+ detail=f"Too many questions: maximum allowed is {MAX_QUESTIONS_PER_REQUEST}.",
97
+ )
98
+
99
+ try:
100
+ model, builder = get_pipeline()
101
+ packed = builder.pack(state=request.state, questions=request.questions)
102
+
103
+ input_token_count = int(packed.input_ids.shape[1])
104
+ if input_token_count > MAX_TOKENS_PER_REQUEST:
105
+ raise HTTPException(
106
+ status_code=status.HTTP_400_BAD_REQUEST,
107
+ detail=f"Prompt length ({input_token_count} tokens) exceeds maximum limit ({MAX_TOKENS_PER_REQUEST}).",
108
+ )
109
+
110
+ answers = model.evaluate_packed(packed)
111
+
112
+ return DecisionResponse(
113
+ model=request.model,
114
+ answers=answers,
115
+ usage=DecisionUsage(
116
+ input_tokens=input_token_count,
117
+ output_tokens=0, # System 1 models don't generate token streams
118
+ ),
119
+ )
120
+ except HTTPException:
121
+ raise
122
+ except ValueError as val_err:
123
+ raise HTTPException(
124
+ status_code=status.HTTP_400_BAD_REQUEST,
125
+ detail=str(val_err),
126
+ )
127
+ except Exception as exc:
128
+ raise HTTPException(
129
+ status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
130
+ detail=f"Internal decision engine error: {exc}",
131
+ )
classone/tokenizer.py ADDED
@@ -0,0 +1,215 @@
1
+ """Prompt formatting, special tokens, and token span tracking for ClassOne.
2
+
3
+ Serializes state and questions into a single structured sequence, tracking the
4
+ exact token index of decision delimiter tokens (<|noul_end|>, <|choice_end|>, etc.)
5
+ for single-pass parallel pooling.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ from dataclasses import dataclass, field
12
+ from typing import Any
13
+
14
+ import torch
15
+ from transformers import PreTrainedTokenizerBase
16
+
17
+ from classone.schemas import ChoiceQuestion, NoulQuestion, Question, ScoreQuestion
18
+
19
+ # Reserved delimiter tokens for ClassOne decision architecture
20
+ SPECIAL_TOKENS = [
21
+ "<|state_start|>",
22
+ "<|state_end|>",
23
+ "<|noul_start|>",
24
+ "<|noul_end|>",
25
+ "<|choice_start|>",
26
+ "<|choice_end|>",
27
+ "<|opt_start|>",
28
+ "<|opt_end|>",
29
+ "<|score_start|>",
30
+ "<|score_end|>",
31
+ "<|level_start|>",
32
+ "<|level_end|>",
33
+ ]
34
+
35
+
36
+ @dataclass
37
+ class QuestionSpans:
38
+ """Token index locations for a single question within the input token sequence."""
39
+
40
+ question_id: str
41
+ question_type: str
42
+ query_token_idx: int
43
+ option_keys: list[str] = field(default_factory=list)
44
+ option_token_indices: list[int] = field(default_factory=list)
45
+
46
+
47
+ @dataclass
48
+ class PackedSequence:
49
+ """Packed prompt tokens and question span indices ready for model forward pass."""
50
+
51
+ text: str
52
+ input_ids: torch.Tensor # shape: [1, seq_len]
53
+ attention_mask: torch.Tensor # shape: [1, seq_len]
54
+ questions: dict[str, QuestionSpans]
55
+
56
+
57
+ class ClassOnePromptBuilder:
58
+ """Builds structured ClassOne decision prompts and maps delimiter token spans."""
59
+
60
+ def __init__(self, tokenizer: PreTrainedTokenizerBase):
61
+ self.tokenizer = tokenizer
62
+ # Add special tokens if not already present
63
+ self.tokenizer.add_special_tokens({"additional_special_tokens": SPECIAL_TOKENS})
64
+ self.special_token_ids = {tok: self.tokenizer.convert_tokens_to_ids(tok) for tok in SPECIAL_TOKENS}
65
+
66
+ def serialize_state(self, state: str | dict[str, Any] | list[Any]) -> str:
67
+ """Serializes state into text."""
68
+ if isinstance(state, str):
69
+ return state.strip()
70
+ return json.dumps(state, indent=2, ensure_ascii=False)
71
+
72
+ def pack(
73
+ self,
74
+ state: str | dict[str, Any] | list[Any],
75
+ questions: dict[str, Question],
76
+ ) -> PackedSequence:
77
+ """Packs state and questions into a single sequence with exact token positions.
78
+
79
+ To guarantee exact token index tracking without tokenizer ambiguity, tokens
80
+ are constructed segment-by-segment.
81
+ """
82
+ state_str = self.serialize_state(state)
83
+
84
+ # Assemble full text representation
85
+ text_parts: list[str] = [f"<|state_start|>\n{state_str}\n<|state_end|>\n"]
86
+
87
+ question_spans_meta: dict[str, QuestionSpans] = {}
88
+
89
+ # Tokenize state prefix first
90
+ prefix_ids = self.tokenizer.encode(
91
+ text_parts[0],
92
+ add_special_tokens=True,
93
+ return_tensors=None,
94
+ )
95
+ current_ids: list[int] = list(prefix_ids)
96
+
97
+ for q_id, q in questions.items():
98
+ if isinstance(q, NoulQuestion):
99
+ q_text = f"<|noul_start|>\nQuestion ID: {q_id}\nInstruction: {q.instructions}\n<|noul_end|>\n"
100
+ seg_ids = self.tokenizer.encode(q_text, add_special_tokens=False, return_tensors=None)
101
+ start_offset = len(current_ids)
102
+ current_ids.extend(seg_ids)
103
+ text_parts.append(q_text)
104
+
105
+ # Locate <|noul_end|> token position in this segment
106
+ end_tok_id = self.special_token_ids["<|noul_end|>"]
107
+ end_indices = [idx for idx, tid in enumerate(seg_ids) if tid == end_tok_id]
108
+ if not end_indices:
109
+ # Fallback to the last token in segment
110
+ query_idx = len(current_ids) - 1
111
+ else:
112
+ query_idx = start_offset + end_indices[-1]
113
+
114
+ question_spans_meta[q_id] = QuestionSpans(
115
+ question_id=q_id,
116
+ question_type="noul",
117
+ query_token_idx=query_idx,
118
+ )
119
+
120
+ elif isinstance(q, ChoiceQuestion):
121
+ # Choice preamble
122
+ choice_header = f"<|choice_start|>\nQuestion ID: {q_id}\nInstruction: {q.instructions}\n"
123
+ header_ids = self.tokenizer.encode(choice_header, add_special_tokens=False, return_tensors=None)
124
+ current_ids.extend(header_ids)
125
+ text_parts.append(choice_header)
126
+
127
+ opt_keys: list[str] = []
128
+ opt_indices: list[int] = []
129
+
130
+ for opt_key, opt_desc in q.criteria.items():
131
+ opt_keys.append(opt_key)
132
+ opt_text = f"<|opt_start|>\nKey: {opt_key}\nCriteria: {opt_desc}\n<|opt_end|>\n"
133
+ opt_ids = self.tokenizer.encode(opt_text, add_special_tokens=False, return_tensors=None)
134
+ start_offset = len(current_ids)
135
+ current_ids.extend(opt_ids)
136
+ text_parts.append(opt_text)
137
+
138
+ end_tok_id = self.special_token_ids["<|opt_end|>"]
139
+ end_idx = [idx for idx, tid in enumerate(opt_ids) if tid == end_tok_id]
140
+ if not end_idx:
141
+ opt_indices.append(len(current_ids) - 1)
142
+ else:
143
+ opt_indices.append(start_offset + end_idx[-1])
144
+
145
+ choice_footer = "<|choice_end|>\n"
146
+ footer_ids = self.tokenizer.encode(choice_footer, add_special_tokens=False, return_tensors=None)
147
+ start_offset = len(current_ids)
148
+ current_ids.extend(footer_ids)
149
+ text_parts.append(choice_footer)
150
+
151
+ end_tok_id = self.special_token_ids["<|choice_end|>"]
152
+ end_idx = [idx for idx, tid in enumerate(footer_ids) if tid == end_tok_id]
153
+ query_idx = start_offset + end_idx[-1] if end_idx else len(current_ids) - 1
154
+
155
+ question_spans_meta[q_id] = QuestionSpans(
156
+ question_id=q_id,
157
+ question_type="choice",
158
+ query_token_idx=query_idx,
159
+ option_keys=opt_keys,
160
+ option_token_indices=opt_indices,
161
+ )
162
+
163
+ elif isinstance(q, ScoreQuestion):
164
+ score_header = f"<|score_start|>\nQuestion ID: {q_id}\nInstruction: {q.instructions}\n"
165
+ header_ids = self.tokenizer.encode(score_header, add_special_tokens=False, return_tensors=None)
166
+ current_ids.extend(header_ids)
167
+ text_parts.append(score_header)
168
+
169
+ level_keys: list[str] = []
170
+ level_indices: list[int] = []
171
+
172
+ for idx, level_desc in enumerate(q.criteria):
173
+ level_key = str(idx + 1)
174
+ level_keys.append(level_key)
175
+ level_text = f"<|level_start|>\nLevel: {level_key}\nCriteria: {level_desc}\n<|level_end|>\n"
176
+ level_ids = self.tokenizer.encode(level_text, add_special_tokens=False, return_tensors=None)
177
+ start_offset = len(current_ids)
178
+ current_ids.extend(level_ids)
179
+ text_parts.append(level_text)
180
+
181
+ end_tok_id = self.special_token_ids["<|level_end|>"]
182
+ end_idx = [idx_pos for idx_pos, tid in enumerate(level_ids) if tid == end_tok_id]
183
+ if not end_idx:
184
+ level_indices.append(len(current_ids) - 1)
185
+ else:
186
+ level_indices.append(start_offset + end_idx[-1])
187
+
188
+ score_footer = "<|score_end|>\n"
189
+ footer_ids = self.tokenizer.encode(score_footer, add_special_tokens=False, return_tensors=None)
190
+ start_offset = len(current_ids)
191
+ current_ids.extend(footer_ids)
192
+ text_parts.append(score_footer)
193
+
194
+ end_tok_id = self.special_token_ids["<|score_end|>"]
195
+ end_idx = [idx for idx, tid in enumerate(footer_ids) if tid == end_tok_id]
196
+ query_idx = start_offset + end_idx[-1] if end_idx else len(current_ids) - 1
197
+
198
+ question_spans_meta[q_id] = QuestionSpans(
199
+ question_id=q_id,
200
+ question_type="score",
201
+ query_token_idx=query_idx,
202
+ option_keys=level_keys,
203
+ option_token_indices=level_indices,
204
+ )
205
+
206
+ full_text = "".join(text_parts)
207
+ input_ids_tensor = torch.tensor([current_ids], dtype=torch.long)
208
+ attention_mask_tensor = torch.ones_like(input_ids_tensor)
209
+
210
+ return PackedSequence(
211
+ text=full_text,
212
+ input_ids=input_ids_tensor,
213
+ attention_mask=attention_mask_tensor,
214
+ questions=question_spans_meta,
215
+ )
@@ -0,0 +1 @@
1
+ """Training and calibration logic (RLCD / proper scoring rules)."""