django-explain-errors 0.11.0__tar.gz → 0.11.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {django_explain_errors-0.11.0/django_explain_errors.egg-info → django_explain_errors-0.11.1}/PKG-INFO +5 -1
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/README.md +4 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1/django_explain_errors.egg-info}/PKG-INFO +5 -1
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/django_explain_errors.egg-info/SOURCES.txt +1 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/middleware.py +53 -6
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/setup.py +1 -1
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_eval_judge.py +158 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_eval_run.py +183 -2
- django_explain_errors-0.11.1/tests/test_token_param_fallback.py +249 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/LICENSE +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/MANIFEST.in +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/django_explain_errors.egg-info/dependency_links.txt +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/django_explain_errors.egg-info/requires.txt +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/django_explain_errors.egg-info/top_level.txt +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/__init__.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/client.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/debug_page.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/management/__init__.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/management/commands/__init__.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/management/commands/build_error_index.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/rag/__init__.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/rag/indexer.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/rag/retriever.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/rag/store.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/sanitize.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/throttle.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/tracebacks.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/setup.cfg +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_async_marking.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_build_error_index.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_client.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_debug_page.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_dedup.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_eval_fixtures.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_language.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_local_embeddings.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_middleware.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_print_stdout.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_process_exception_split.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_rag.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_sanitize.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_signals.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_throttle.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_tracebacks.py +0 -0
- {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_truncation.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: django-explain-errors
|
|
3
|
-
Version: 0.11.
|
|
3
|
+
Version: 0.11.1
|
|
4
4
|
Summary: Django middleware that explains unhandled exceptions in DEBUG using an LLM, optionally grounded in your own project source via a local vector index. Works with OpenAI, Claude, or any OpenAI-compatible endpoint including local models.
|
|
5
5
|
Home-page: https://github.com/topunix/django-explain-errors
|
|
6
6
|
Author: topunix
|
|
@@ -272,6 +272,10 @@ catches the exception internally, so it never becomes an unhandled exception, bu
|
|
|
272
272
|
than relying on `got_request_exception`. That event is unrelated to whatever error the
|
|
273
273
|
developer is actually investigating.
|
|
274
274
|
|
|
275
|
+
OpenAI reasoning models are supported. They reject `max_tokens`, so when the API rejects it
|
|
276
|
+
the middleware retries once with `max_completion_tokens` (same value, `OPENAI_MAX_TOKENS`)
|
|
277
|
+
and keeps using that for later calls. Other providers keep receiving `max_tokens`.
|
|
278
|
+
|
|
275
279
|
## Configuration
|
|
276
280
|
|
|
277
281
|
| Setting / variable | Required | Description |
|
|
@@ -220,6 +220,10 @@ catches the exception internally, so it never becomes an unhandled exception, bu
|
|
|
220
220
|
than relying on `got_request_exception`. That event is unrelated to whatever error the
|
|
221
221
|
developer is actually investigating.
|
|
222
222
|
|
|
223
|
+
OpenAI reasoning models are supported. They reject `max_tokens`, so when the API rejects it
|
|
224
|
+
the middleware retries once with `max_completion_tokens` (same value, `OPENAI_MAX_TOKENS`)
|
|
225
|
+
and keeps using that for later calls. Other providers keep receiving `max_tokens`.
|
|
226
|
+
|
|
223
227
|
## Configuration
|
|
224
228
|
|
|
225
229
|
| Setting / variable | Required | Description |
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: django-explain-errors
|
|
3
|
-
Version: 0.11.
|
|
3
|
+
Version: 0.11.1
|
|
4
4
|
Summary: Django middleware that explains unhandled exceptions in DEBUG using an LLM, optionally grounded in your own project source via a local vector index. Works with OpenAI, Claude, or any OpenAI-compatible endpoint including local models.
|
|
5
5
|
Home-page: https://github.com/topunix/django-explain-errors
|
|
6
6
|
Author: topunix
|
|
@@ -272,6 +272,10 @@ catches the exception internally, so it never becomes an unhandled exception, bu
|
|
|
272
272
|
than relying on `got_request_exception`. That event is unrelated to whatever error the
|
|
273
273
|
developer is actually investigating.
|
|
274
274
|
|
|
275
|
+
OpenAI reasoning models are supported. They reject `max_tokens`, so when the API rejects it
|
|
276
|
+
the middleware retries once with `max_completion_tokens` (same value, `OPENAI_MAX_TOKENS`)
|
|
277
|
+
and keeps using that for later calls. Other providers keep receiving `max_tokens`.
|
|
278
|
+
|
|
275
279
|
## Configuration
|
|
276
280
|
|
|
277
281
|
| Setting / variable | Required | Description |
|
|
@@ -3,6 +3,7 @@ import logging
|
|
|
3
3
|
import threading
|
|
4
4
|
from collections import OrderedDict
|
|
5
5
|
|
|
6
|
+
import openai
|
|
6
7
|
from django.conf import settings
|
|
7
8
|
from django.http import JsonResponse
|
|
8
9
|
from asgiref.sync import markcoroutinefunction, sync_to_async
|
|
@@ -81,6 +82,9 @@ class ExplainErrorsMiddleware:
|
|
|
81
82
|
# above this one runs in sync mode and receives a coroutine.
|
|
82
83
|
markcoroutinefunction(self)
|
|
83
84
|
self.openai_client = None
|
|
85
|
+
# Output-cap parameter sent to the model. Reasoning models reject
|
|
86
|
+
# max_tokens; _create_completion switches this once, per instance.
|
|
87
|
+
self._token_param = "max_tokens"
|
|
84
88
|
self.throttle = None
|
|
85
89
|
self.dedup_enabled = False
|
|
86
90
|
self._explanation_cache = OrderedDict()
|
|
@@ -251,6 +255,39 @@ class ExplainErrorsMiddleware:
|
|
|
251
255
|
|
|
252
256
|
return prompt
|
|
253
257
|
|
|
258
|
+
def _create_completion(self, messages):
|
|
259
|
+
"""Send the chat completion, adapting to the model's output-cap parameter.
|
|
260
|
+
|
|
261
|
+
Sends max_tokens by default. If the API rejects exactly that parameter
|
|
262
|
+
(OpenAI reasoning models require max_completion_tokens), retries once
|
|
263
|
+
with max_completion_tokens and remembers the choice on this instance.
|
|
264
|
+
Any other error, or a failed retry, propagates to the caller.
|
|
265
|
+
"""
|
|
266
|
+
|
|
267
|
+
def create(token_param):
|
|
268
|
+
return self.openai_client.chat.completions.create(
|
|
269
|
+
model=self.model,
|
|
270
|
+
messages=messages,
|
|
271
|
+
**{token_param: self.max_tokens},
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
try:
|
|
275
|
+
return create(self._token_param)
|
|
276
|
+
except openai.BadRequestError as exc:
|
|
277
|
+
if self._token_param != "max_tokens" or not (
|
|
278
|
+
getattr(exc, "param", None) == "max_tokens"
|
|
279
|
+
and getattr(exc, "code", None) == "unsupported_parameter"
|
|
280
|
+
):
|
|
281
|
+
raise
|
|
282
|
+
response = create("max_completion_tokens")
|
|
283
|
+
self._token_param = "max_completion_tokens"
|
|
284
|
+
logger.info(
|
|
285
|
+
"explain_errors: model %s rejected max_tokens; using "
|
|
286
|
+
"max_completion_tokens instead",
|
|
287
|
+
self.model,
|
|
288
|
+
)
|
|
289
|
+
return response
|
|
290
|
+
|
|
254
291
|
def _generate_explanation(self, prompt):
|
|
255
292
|
"""Call the model and return the explanation text, or None on failure.
|
|
256
293
|
|
|
@@ -260,13 +297,11 @@ class ExplainErrorsMiddleware:
|
|
|
260
297
|
explanation = None
|
|
261
298
|
try:
|
|
262
299
|
# Call model API
|
|
263
|
-
response = self.
|
|
264
|
-
|
|
265
|
-
messages=[
|
|
300
|
+
response = self._create_completion(
|
|
301
|
+
[
|
|
266
302
|
{"role": "system", "content": self.system_prompt},
|
|
267
303
|
{"role": "user", "content": prompt},
|
|
268
|
-
]
|
|
269
|
-
max_tokens=self.max_tokens,
|
|
304
|
+
]
|
|
270
305
|
)
|
|
271
306
|
explanation = response.choices[0].message.content
|
|
272
307
|
usage = getattr(response, "usage", None)
|
|
@@ -276,11 +311,23 @@ class ExplainErrorsMiddleware:
|
|
|
276
311
|
getattr(usage, "completion_tokens", None),
|
|
277
312
|
)
|
|
278
313
|
if response.choices[0].finish_reason == "length":
|
|
314
|
+
hint = ""
|
|
315
|
+
if self._token_param == "max_completion_tokens":
|
|
316
|
+
hint = (
|
|
317
|
+
" (reasoning tokens count against this limit; raise it "
|
|
318
|
+
"if explanations come back short or empty)"
|
|
319
|
+
)
|
|
279
320
|
logger.warning(
|
|
280
|
-
"explain_errors: explanation truncated by OPENAI_MAX_TOKENS=%s",
|
|
321
|
+
"explain_errors: explanation truncated by OPENAI_MAX_TOKENS=%s%s",
|
|
281
322
|
self.max_tokens,
|
|
323
|
+
hint,
|
|
282
324
|
)
|
|
283
325
|
|
|
326
|
+
if not explanation or not explanation.strip():
|
|
327
|
+
# Reasoning can consume the whole budget and leave no text.
|
|
328
|
+
explanation = None
|
|
329
|
+
raise ValueError("the model returned an empty response")
|
|
330
|
+
|
|
284
331
|
if getattr(settings, "EXPLAIN_ERRORS_PRINT_STDOUT", True):
|
|
285
332
|
print(f"Error explanation ({self.model}):\n", explanation)
|
|
286
333
|
except Exception as e:
|
|
@@ -10,7 +10,7 @@ with open('README.md', encoding='utf-8') as f:
|
|
|
10
10
|
|
|
11
11
|
setup(
|
|
12
12
|
name='django-explain-errors',
|
|
13
|
-
version='0.11.
|
|
13
|
+
version='0.11.1',
|
|
14
14
|
packages=find_packages(exclude=['tests', 'tests.*', 'evals', 'evals.*']),
|
|
15
15
|
description='Django middleware that explains unhandled exceptions in DEBUG using an LLM, optionally grounded in your own project source via a local vector index. Works with OpenAI, Claude, or any OpenAI-compatible endpoint including local models.',
|
|
16
16
|
long_description_content_type='text/markdown',
|
|
@@ -391,3 +391,161 @@ class CompareClaimsGuardTest(unittest.TestCase):
|
|
|
391
391
|
)
|
|
392
392
|
|
|
393
393
|
self.assertFalse(result["judge_failure"])
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _openai_error(cls, status=None):
|
|
397
|
+
"""An instance of the openai exception class without constructing its
|
|
398
|
+
httpx request/response (which httpx package openai imports varies by
|
|
399
|
+
openai version).
|
|
400
|
+
"""
|
|
401
|
+
exc = Exception.__new__(cls)
|
|
402
|
+
exc.args = ("boom",)
|
|
403
|
+
if status is not None:
|
|
404
|
+
exc.status_code = status
|
|
405
|
+
return exc
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _flaky_client(errors, then=VALID_RESPONSE):
|
|
409
|
+
"""Raises each of `errors` in turn, then returns `then`."""
|
|
410
|
+
client = MagicMock()
|
|
411
|
+
ok = MagicMock(
|
|
412
|
+
choices=[MagicMock(message=MagicMock(content=then))],
|
|
413
|
+
usage=MagicMock(prompt_tokens=7, completion_tokens=3),
|
|
414
|
+
)
|
|
415
|
+
client.chat.completions.create.side_effect = list(errors) + [ok]
|
|
416
|
+
return client
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
class CompareRetryTest(unittest.TestCase):
|
|
420
|
+
|
|
421
|
+
def test_pre_drawn_position_is_used_and_rng_untouched(self):
|
|
422
|
+
fixture = _make_fixture()
|
|
423
|
+
client = _client_returning(VALID_RESPONSE)
|
|
424
|
+
|
|
425
|
+
class ExplodingRng:
|
|
426
|
+
def random(self):
|
|
427
|
+
raise AssertionError("rng must not be drawn when rag_on_is_a is given")
|
|
428
|
+
|
|
429
|
+
result = compare(
|
|
430
|
+
client, "m", "tb", fixture, "src", "off", "on",
|
|
431
|
+
rng=ExplodingRng(), rag_on_is_a=False,
|
|
432
|
+
)
|
|
433
|
+
|
|
434
|
+
self.assertFalse(result["rag_on_is_a"])
|
|
435
|
+
|
|
436
|
+
def test_rate_limit_is_retried_and_counted(self):
|
|
437
|
+
import openai
|
|
438
|
+
|
|
439
|
+
fixture = _make_fixture()
|
|
440
|
+
errors = [_openai_error(openai.RateLimitError, 429)] * 2
|
|
441
|
+
client = _flaky_client(errors)
|
|
442
|
+
sleeps = []
|
|
443
|
+
|
|
444
|
+
result = compare(
|
|
445
|
+
client, "m", "tb", fixture, "src", "off", "on",
|
|
446
|
+
rng=_FixedRng(0.1), sleep=sleeps.append,
|
|
447
|
+
)
|
|
448
|
+
|
|
449
|
+
self.assertFalse(result["judge_failure"])
|
|
450
|
+
self.assertEqual(result["retries"], 2)
|
|
451
|
+
self.assertEqual(len(sleeps), 2)
|
|
452
|
+
self.assertEqual(client.chat.completions.create.call_count, 3)
|
|
453
|
+
self.assertEqual(result["prompt_tokens"], 7)
|
|
454
|
+
|
|
455
|
+
def test_backoff_grows_and_is_capped(self):
|
|
456
|
+
import openai
|
|
457
|
+
|
|
458
|
+
from evals import judge
|
|
459
|
+
|
|
460
|
+
fixture = _make_fixture()
|
|
461
|
+
client = _flaky_client([_openai_error(openai.InternalServerError, 503)] * 4)
|
|
462
|
+
ceilings = []
|
|
463
|
+
|
|
464
|
+
class RecordingJitter:
|
|
465
|
+
def uniform(self, low, high):
|
|
466
|
+
ceilings.append(high)
|
|
467
|
+
return high
|
|
468
|
+
|
|
469
|
+
original = judge._jitter_rng
|
|
470
|
+
judge._jitter_rng = RecordingJitter()
|
|
471
|
+
self.addCleanup(setattr, judge, "_jitter_rng", original)
|
|
472
|
+
|
|
473
|
+
compare(
|
|
474
|
+
client, "m", "tb", fixture, "src", "off", "on",
|
|
475
|
+
rng=_FixedRng(0.1), max_attempts=5, sleep=lambda s: None,
|
|
476
|
+
)
|
|
477
|
+
|
|
478
|
+
self.assertEqual(ceilings, [1.0, 2.0, 4.0, 8.0])
|
|
479
|
+
|
|
480
|
+
def test_timeout_and_connection_errors_are_retried(self):
|
|
481
|
+
import openai
|
|
482
|
+
|
|
483
|
+
fixture = _make_fixture()
|
|
484
|
+
client = _flaky_client(
|
|
485
|
+
[_openai_error(openai.APITimeoutError), _openai_error(openai.APIConnectionError)]
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
result = compare(
|
|
489
|
+
client, "m", "tb", fixture, "src", "off", "on",
|
|
490
|
+
rng=_FixedRng(0.1), sleep=lambda s: None,
|
|
491
|
+
)
|
|
492
|
+
|
|
493
|
+
self.assertFalse(result["judge_failure"])
|
|
494
|
+
self.assertEqual(result["retries"], 2)
|
|
495
|
+
|
|
496
|
+
def test_exhausted_retries_is_a_judge_failure_with_retries_recorded(self):
|
|
497
|
+
import openai
|
|
498
|
+
|
|
499
|
+
fixture = _make_fixture()
|
|
500
|
+
client = MagicMock()
|
|
501
|
+
client.chat.completions.create.side_effect = _openai_error(openai.RateLimitError, 429)
|
|
502
|
+
|
|
503
|
+
result = compare(
|
|
504
|
+
client, "m", "tb", fixture, "src", "off", "on",
|
|
505
|
+
rng=_FixedRng(0.1), max_attempts=3, sleep=lambda s: None,
|
|
506
|
+
)
|
|
507
|
+
|
|
508
|
+
self.assertTrue(result["judge_failure"])
|
|
509
|
+
self.assertEqual(result["retries"], 2)
|
|
510
|
+
self.assertEqual(client.chat.completions.create.call_count, 3)
|
|
511
|
+
self.assertIn("boom", result["error"])
|
|
512
|
+
self.assertIn("rag_on_is_a", result)
|
|
513
|
+
|
|
514
|
+
def test_non_transient_error_is_not_retried(self):
|
|
515
|
+
import openai
|
|
516
|
+
|
|
517
|
+
fixture = _make_fixture()
|
|
518
|
+
client = MagicMock()
|
|
519
|
+
client.chat.completions.create.side_effect = _openai_error(
|
|
520
|
+
openai.AuthenticationError, 401
|
|
521
|
+
)
|
|
522
|
+
|
|
523
|
+
result = compare(
|
|
524
|
+
client, "m", "tb", fixture, "src", "off", "on",
|
|
525
|
+
rng=_FixedRng(0.1), sleep=lambda s: self.fail("must not sleep"),
|
|
526
|
+
)
|
|
527
|
+
|
|
528
|
+
self.assertTrue(result["judge_failure"])
|
|
529
|
+
self.assertEqual(result["retries"], 0)
|
|
530
|
+
self.assertEqual(client.chat.completions.create.call_count, 1)
|
|
531
|
+
|
|
532
|
+
def test_malformed_response_is_not_retried(self):
|
|
533
|
+
fixture = _make_fixture()
|
|
534
|
+
client = _client_returning("not json")
|
|
535
|
+
|
|
536
|
+
result = compare(
|
|
537
|
+
client, "m", "tb", fixture, "src", "off", "on",
|
|
538
|
+
rng=_FixedRng(0.1), sleep=lambda s: self.fail("must not sleep"),
|
|
539
|
+
)
|
|
540
|
+
|
|
541
|
+
self.assertTrue(result["judge_failure"])
|
|
542
|
+
self.assertEqual(result["retries"], 0)
|
|
543
|
+
self.assertEqual(client.chat.completions.create.call_count, 1)
|
|
544
|
+
|
|
545
|
+
def test_clean_call_records_zero_retries(self):
|
|
546
|
+
fixture = _make_fixture()
|
|
547
|
+
client = _client_returning(VALID_RESPONSE)
|
|
548
|
+
|
|
549
|
+
result = compare(client, "m", "tb", fixture, "src", "off", "on", rng=_FixedRng(0.1))
|
|
550
|
+
|
|
551
|
+
self.assertEqual(result["retries"], 0)
|
|
@@ -8,9 +8,22 @@ import json
|
|
|
8
8
|
import os
|
|
9
9
|
import subprocess
|
|
10
10
|
import sys
|
|
11
|
+
import random
|
|
12
|
+
import runpy
|
|
13
|
+
import threading
|
|
14
|
+
import time
|
|
11
15
|
import unittest
|
|
12
|
-
|
|
13
|
-
|
|
16
|
+
from unittest.mock import patch
|
|
17
|
+
|
|
18
|
+
from evals.fixtures import FIXTURES_BY_NAME
|
|
19
|
+
from evals.run import (
|
|
20
|
+
estimate_cost_usd,
|
|
21
|
+
judge_all,
|
|
22
|
+
tally_judgments,
|
|
23
|
+
tally_latency,
|
|
24
|
+
tally_retries,
|
|
25
|
+
tally_usage,
|
|
26
|
+
)
|
|
14
27
|
|
|
15
28
|
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
16
29
|
|
|
@@ -197,6 +210,11 @@ class TallyLatencyAndUsageTest(unittest.TestCase):
|
|
|
197
210
|
cost = estimate_cost_usd("gpt-4o-mini", 1000, 1000)
|
|
198
211
|
self.assertAlmostEqual(cost, 0.00015 + 0.0006)
|
|
199
212
|
|
|
213
|
+
def test_estimate_cost_for_gpt_6_luna(self):
|
|
214
|
+
cost = estimate_cost_usd("gpt-6-luna", 1000, 1000)
|
|
215
|
+
self.assertIsNotNone(cost)
|
|
216
|
+
self.assertAlmostEqual(cost, 0.0001 + 0.0005)
|
|
217
|
+
|
|
200
218
|
|
|
201
219
|
class BuildJudgeSourceTest(unittest.TestCase):
|
|
202
220
|
"""Runs tests/_eval_judge_source_helper.py in a subprocess -- see its
|
|
@@ -279,6 +297,161 @@ class BuildJudgeSourceTest(unittest.TestCase):
|
|
|
279
297
|
self.assertFalse(any(label.endswith(".html") for label in labels))
|
|
280
298
|
|
|
281
299
|
|
|
300
|
+
class JudgeAllConcurrencyTest(unittest.TestCase):
|
|
301
|
+
"""judge_all with a fake judge: position draws happen up front, in
|
|
302
|
+
order, independent of concurrency, and results keep job order.
|
|
303
|
+
"""
|
|
304
|
+
|
|
305
|
+
FIXTURE_NAMES = ("none_attribute", "str_recursion", "bad_lookup")
|
|
306
|
+
RUNS = 3
|
|
307
|
+
SEED = 20260711
|
|
308
|
+
|
|
309
|
+
def _calls(self):
|
|
310
|
+
return [
|
|
311
|
+
{
|
|
312
|
+
"fixture": name,
|
|
313
|
+
"rag_enabled": rag,
|
|
314
|
+
"run_index": run,
|
|
315
|
+
"traceback": "tb",
|
|
316
|
+
"explanation": f"{name}/{rag}/{run}",
|
|
317
|
+
}
|
|
318
|
+
for name in self.FIXTURE_NAMES
|
|
319
|
+
for rag in (False, True)
|
|
320
|
+
for run in range(self.RUNS)
|
|
321
|
+
]
|
|
322
|
+
|
|
323
|
+
def _run(self, concurrency, scramble=True):
|
|
324
|
+
"""Returns (judgments, draws_before_first_call, call_order)."""
|
|
325
|
+
fixtures = [FIXTURES_BY_NAME[n] for n in self.FIXTURE_NAMES]
|
|
326
|
+
draws = []
|
|
327
|
+
lock = threading.Lock()
|
|
328
|
+
state = {"draws_at_first_call": None}
|
|
329
|
+
call_order = []
|
|
330
|
+
|
|
331
|
+
class CountingRandom(random.Random):
|
|
332
|
+
def random(self):
|
|
333
|
+
value = super().random()
|
|
334
|
+
draws.append(value)
|
|
335
|
+
return value
|
|
336
|
+
|
|
337
|
+
def fake_compare(client, model, tb, fixture, source, off, on, rag_on_is_a=None, **kw):
|
|
338
|
+
with lock:
|
|
339
|
+
if state["draws_at_first_call"] is None:
|
|
340
|
+
state["draws_at_first_call"] = len(draws)
|
|
341
|
+
call_order.append((fixture.name, off))
|
|
342
|
+
if scramble:
|
|
343
|
+
# Finish out of submission order: later jobs return sooner.
|
|
344
|
+
time.sleep(0.02 if off.endswith("/0") else 0.0)
|
|
345
|
+
return {
|
|
346
|
+
"fixture": fixture.name,
|
|
347
|
+
"run_index": None,
|
|
348
|
+
"rag_on_is_a": rag_on_is_a,
|
|
349
|
+
"off": off,
|
|
350
|
+
"on": on,
|
|
351
|
+
"retries": 0,
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
with patch("evals.judge.get_judge_client", return_value=object()), patch(
|
|
355
|
+
"evals.judge.get_judge_model", return_value="m"
|
|
356
|
+
), patch("evals.judge.compare", side_effect=fake_compare), patch(
|
|
357
|
+
"evals.run._build_judge_source", return_value=[]
|
|
358
|
+
), patch("evals.run.random.Random", CountingRandom):
|
|
359
|
+
judgments = judge_all(
|
|
360
|
+
fixtures, self._calls(), {}, concurrency=concurrency, seed=self.SEED
|
|
361
|
+
)
|
|
362
|
+
return judgments, state["draws_at_first_call"], call_order
|
|
363
|
+
|
|
364
|
+
def test_all_draws_happen_before_the_first_judge_call(self):
|
|
365
|
+
total = len(self.FIXTURE_NAMES) * self.RUNS
|
|
366
|
+
for n in (1, 4, 8):
|
|
367
|
+
_, draws_at_first_call, _ = self._run(n)
|
|
368
|
+
self.assertEqual(draws_at_first_call, total, f"N={n}")
|
|
369
|
+
|
|
370
|
+
def test_positions_match_the_seeded_rng_in_fixture_then_run_order(self):
|
|
371
|
+
reference = random.Random(self.SEED)
|
|
372
|
+
expected = [
|
|
373
|
+
reference.random() < 0.5
|
|
374
|
+
for _ in range(len(self.FIXTURE_NAMES) * self.RUNS)
|
|
375
|
+
]
|
|
376
|
+
for n in (1, 2, 8, 32):
|
|
377
|
+
judgments, _, _ = self._run(n)
|
|
378
|
+
self.assertEqual([j["rag_on_is_a"] for j in judgments], expected, f"N={n}")
|
|
379
|
+
|
|
380
|
+
def test_results_keep_comparison_order_even_when_completion_order_differs(self):
|
|
381
|
+
expected_order = [
|
|
382
|
+
(name, run) for name in self.FIXTURE_NAMES for run in range(self.RUNS)
|
|
383
|
+
]
|
|
384
|
+
for n in (1, 8):
|
|
385
|
+
judgments, _, _ = self._run(n)
|
|
386
|
+
self.assertEqual(
|
|
387
|
+
[(j["fixture"], j["run_index"]) for j in judgments], expected_order, f"N={n}"
|
|
388
|
+
)
|
|
389
|
+
self.assertEqual(
|
|
390
|
+
[j["off"] for j in judgments],
|
|
391
|
+
[f"{name}/False/{run}" for name, run in expected_order],
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
def test_concurrency_one_calls_in_order(self):
|
|
395
|
+
_, _, call_order = self._run(1)
|
|
396
|
+
self.assertEqual(
|
|
397
|
+
[name for name, _ in call_order],
|
|
398
|
+
[name for name in self.FIXTURE_NAMES for _ in range(self.RUNS)],
|
|
399
|
+
)
|
|
400
|
+
|
|
401
|
+
def test_concurrency_actually_overlaps_calls(self):
|
|
402
|
+
active = {"now": 0, "peak": 0}
|
|
403
|
+
lock = threading.Lock()
|
|
404
|
+
|
|
405
|
+
def fake_compare(*args, **kwargs):
|
|
406
|
+
with lock:
|
|
407
|
+
active["now"] += 1
|
|
408
|
+
active["peak"] = max(active["peak"], active["now"])
|
|
409
|
+
time.sleep(0.05)
|
|
410
|
+
with lock:
|
|
411
|
+
active["now"] -= 1
|
|
412
|
+
return {"fixture": "x", "retries": 0}
|
|
413
|
+
|
|
414
|
+
fixtures = [FIXTURES_BY_NAME[n] for n in self.FIXTURE_NAMES]
|
|
415
|
+
with patch("evals.judge.get_judge_client", return_value=object()), patch(
|
|
416
|
+
"evals.judge.get_judge_model", return_value="m"
|
|
417
|
+
), patch("evals.judge.compare", side_effect=fake_compare), patch(
|
|
418
|
+
"evals.run._build_judge_source", return_value=[]
|
|
419
|
+
):
|
|
420
|
+
judge_all(fixtures, self._calls(), {}, concurrency=4, seed=1)
|
|
421
|
+
self.assertGreater(active["peak"], 1)
|
|
422
|
+
self.assertLessEqual(active["peak"], 4)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
class TallyRetriesTest(unittest.TestCase):
|
|
426
|
+
|
|
427
|
+
def test_sums_per_judgment_retries_and_tolerates_missing(self):
|
|
428
|
+
self.assertEqual(
|
|
429
|
+
tally_retries([{"retries": 2}, {"retries": 0}, {}, {"retries": None}, {"retries": 1}]),
|
|
430
|
+
3,
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
class FixtureSettingsModelTest(unittest.TestCase):
|
|
435
|
+
"""The fixture app's settings resolve OPENAI_MODEL from the environment."""
|
|
436
|
+
|
|
437
|
+
SETTINGS_PATH = os.path.join(REPO_ROOT, "evals", "fixture_app", "settings.py")
|
|
438
|
+
|
|
439
|
+
def _model(self, **env):
|
|
440
|
+
with patch.dict(os.environ, env):
|
|
441
|
+
if "OPENAI_MODEL" not in env:
|
|
442
|
+
os.environ.pop("OPENAI_MODEL", None)
|
|
443
|
+
return runpy.run_path(self.SETTINGS_PATH)["OPENAI_MODEL"]
|
|
444
|
+
|
|
445
|
+
def test_reads_env_var(self):
|
|
446
|
+
self.assertEqual(self._model(OPENAI_MODEL="some-other-model"), "some-other-model")
|
|
447
|
+
|
|
448
|
+
def test_unset_falls_back_to_default(self):
|
|
449
|
+
self.assertEqual(self._model(), "gpt-4o-mini")
|
|
450
|
+
|
|
451
|
+
def test_empty_string_falls_back_to_default(self):
|
|
452
|
+
self.assertEqual(self._model(OPENAI_MODEL=""), "gpt-4o-mini")
|
|
453
|
+
|
|
454
|
+
|
|
282
455
|
class EvalHarnessEndToEndTest(unittest.TestCase):
|
|
283
456
|
"""Runs evals.run.main() in a subprocess with the generator (chat +
|
|
284
457
|
embeddings) and the judge fully mocked
|
|
@@ -326,10 +499,17 @@ class EvalHarnessEndToEndTest(unittest.TestCase):
|
|
|
326
499
|
"generator_model",
|
|
327
500
|
"judge_model",
|
|
328
501
|
"runs",
|
|
502
|
+
"seed",
|
|
503
|
+
"timing",
|
|
504
|
+
"judge_retries_total",
|
|
329
505
|
"calls",
|
|
330
506
|
"judgments",
|
|
331
507
|
):
|
|
332
508
|
self.assertIn(key, data)
|
|
509
|
+
self.assertEqual(data["judge_retries_total"], 0)
|
|
510
|
+
self.assertEqual(data["timing"]["judge_concurrency"], 8)
|
|
511
|
+
self.assertGreaterEqual(data["timing"]["generator_seconds"], 0)
|
|
512
|
+
self.assertGreaterEqual(data["timing"]["judge_seconds"], 0)
|
|
333
513
|
|
|
334
514
|
self.assertEqual(data["generator_model"], "gpt-4o-mini")
|
|
335
515
|
self.assertEqual(data["judge_model"], "judge-test-model")
|
|
@@ -369,6 +549,7 @@ class EvalHarnessEndToEndTest(unittest.TestCase):
|
|
|
369
549
|
"b",
|
|
370
550
|
"reasoning",
|
|
371
551
|
"run_index",
|
|
552
|
+
"retries",
|
|
372
553
|
}
|
|
373
554
|
for judgment in data["judgments"]:
|
|
374
555
|
self.assertTrue(judgment_keys.issubset(judgment.keys()), judgment)
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
from unittest.mock import MagicMock, patch
|
|
2
|
+
|
|
3
|
+
import openai
|
|
4
|
+
from django.test import AsyncClient, Client, SimpleTestCase, override_settings
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def _bad_request(param, code):
|
|
8
|
+
body = {
|
|
9
|
+
"message": "Unsupported parameter",
|
|
10
|
+
"type": "invalid_request_error",
|
|
11
|
+
"param": param,
|
|
12
|
+
"code": code,
|
|
13
|
+
}
|
|
14
|
+
response = MagicMock(status_code=400)
|
|
15
|
+
return openai.BadRequestError("Unsupported parameter", response=response, body=body)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _ok_response(text="Mocked explanation.", finish_reason="stop"):
|
|
19
|
+
return MagicMock(
|
|
20
|
+
choices=[
|
|
21
|
+
MagicMock(message=MagicMock(content=text), finish_reason=finish_reason)
|
|
22
|
+
]
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _rejects_max_tokens(**kwargs):
|
|
27
|
+
if "max_tokens" in kwargs:
|
|
28
|
+
raise _bad_request("max_tokens", "unsupported_parameter")
|
|
29
|
+
return _ok_response()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@override_settings(
|
|
33
|
+
DEBUG=True,
|
|
34
|
+
OPENAI_API_KEY="test-key",
|
|
35
|
+
OPENAI_MAX_TOKENS=321,
|
|
36
|
+
EXPLAIN_ERRORS_DEDUP=False,
|
|
37
|
+
ROOT_URLCONF="tests.urls_debug_page",
|
|
38
|
+
MIDDLEWARE=["explain_errors.middleware.ExplainErrorsMiddleware"],
|
|
39
|
+
)
|
|
40
|
+
class TokenParamFallbackBase(SimpleTestCase):
|
|
41
|
+
def setUp(self):
|
|
42
|
+
patcher = patch("openai.OpenAI")
|
|
43
|
+
mock_cls = patcher.start()
|
|
44
|
+
self.addCleanup(patcher.stop)
|
|
45
|
+
self.create = MagicMock(return_value=_ok_response())
|
|
46
|
+
mock_cls.return_value.chat.completions.create = self.create
|
|
47
|
+
|
|
48
|
+
def sent(self):
|
|
49
|
+
return [call.kwargs for call in self.create.call_args_list]
|
|
50
|
+
|
|
51
|
+
def assertTokenParam(self, kwargs, expected):
|
|
52
|
+
other = (
|
|
53
|
+
"max_completion_tokens" if expected == "max_tokens" else "max_tokens"
|
|
54
|
+
)
|
|
55
|
+
self.assertEqual(kwargs[expected], 321)
|
|
56
|
+
self.assertNotIn(other, kwargs)
|
|
57
|
+
|
|
58
|
+
def assertEmptyContentHandled(self, mock_print, response):
|
|
59
|
+
firsts = [call.args[0] for call in mock_print.call_args_list if call.args]
|
|
60
|
+
self.assertEqual(
|
|
61
|
+
firsts.count("Failed to get an explanation from the model API:"), 1
|
|
62
|
+
)
|
|
63
|
+
self.assertFalse([f for f in firsts if str(f).startswith("Error explanation")])
|
|
64
|
+
self.assertNotIn('id="explain-errors"', response.content.decode())
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class SyncTokenParamFallbackTest(TokenParamFallbackBase):
|
|
68
|
+
def trigger(self):
|
|
69
|
+
client = getattr(self, "_client", None)
|
|
70
|
+
if client is None:
|
|
71
|
+
client = self._client = Client(raise_request_exception=False)
|
|
72
|
+
return client.get("/boom/")
|
|
73
|
+
|
|
74
|
+
def test_default_sends_max_tokens_only(self):
|
|
75
|
+
self.trigger()
|
|
76
|
+
calls = self.sent()
|
|
77
|
+
self.assertEqual(len(calls), 1)
|
|
78
|
+
self.assertTokenParam(calls[0], "max_tokens")
|
|
79
|
+
|
|
80
|
+
def test_rejection_retries_once_with_max_completion_tokens(self):
|
|
81
|
+
self.create.side_effect = _rejects_max_tokens
|
|
82
|
+
with self.assertLogs("explain_errors.middleware", level="INFO") as logs:
|
|
83
|
+
response = self.trigger()
|
|
84
|
+
calls = self.sent()
|
|
85
|
+
self.assertEqual(len(calls), 2)
|
|
86
|
+
self.assertTokenParam(calls[0], "max_tokens")
|
|
87
|
+
self.assertTokenParam(calls[1], "max_completion_tokens")
|
|
88
|
+
self.assertIn("Mocked explanation.", response.content.decode())
|
|
89
|
+
self.assertEqual(len(logs.records), 1)
|
|
90
|
+
self.assertIn("max_tokens", logs.records[0].getMessage())
|
|
91
|
+
self.assertIn("max_completion_tokens", logs.records[0].getMessage())
|
|
92
|
+
|
|
93
|
+
def test_next_call_uses_max_completion_tokens_directly(self):
|
|
94
|
+
self.create.side_effect = _rejects_max_tokens
|
|
95
|
+
self.trigger()
|
|
96
|
+
self.create.reset_mock()
|
|
97
|
+
self.trigger()
|
|
98
|
+
calls = self.sent()
|
|
99
|
+
self.assertEqual(len(calls), 1)
|
|
100
|
+
self.assertTokenParam(calls[0], "max_completion_tokens")
|
|
101
|
+
|
|
102
|
+
def test_other_param_does_not_retry(self):
|
|
103
|
+
self.create.side_effect = _bad_request("temperature", "unsupported_parameter")
|
|
104
|
+
self.trigger()
|
|
105
|
+
self.assertEqual(len(self.sent()), 1)
|
|
106
|
+
|
|
107
|
+
def test_other_code_does_not_retry(self):
|
|
108
|
+
self.create.side_effect = _bad_request("max_tokens", "invalid_value")
|
|
109
|
+
self.trigger()
|
|
110
|
+
self.assertEqual(len(self.sent()), 1)
|
|
111
|
+
|
|
112
|
+
def test_failed_retry_does_not_loop_and_uses_failure_handling(self):
|
|
113
|
+
self.create.side_effect = [
|
|
114
|
+
_bad_request("max_tokens", "unsupported_parameter"),
|
|
115
|
+
RuntimeError("api down"),
|
|
116
|
+
]
|
|
117
|
+
with patch("builtins.print") as mock_print:
|
|
118
|
+
self.trigger()
|
|
119
|
+
self.assertEqual(len(self.sent()), 2)
|
|
120
|
+
self.assertIn(
|
|
121
|
+
"Failed to get an explanation from the model API:",
|
|
122
|
+
[call.args[0] for call in mock_print.call_args_list],
|
|
123
|
+
)
|
|
124
|
+
# The switch was not remembered after a failed retry.
|
|
125
|
+
self.create.reset_mock()
|
|
126
|
+
self.create.side_effect = None
|
|
127
|
+
self.create.return_value = _ok_response()
|
|
128
|
+
self.trigger()
|
|
129
|
+
self.assertTokenParam(self.sent()[0], "max_tokens")
|
|
130
|
+
|
|
131
|
+
def test_length_warning_hints_at_reasoning_tokens_after_fallback(self):
|
|
132
|
+
self.create.side_effect = _rejects_max_tokens
|
|
133
|
+
self.trigger()
|
|
134
|
+
self.create.side_effect = None
|
|
135
|
+
self.create.return_value = _ok_response(finish_reason="length")
|
|
136
|
+
with self.assertLogs("explain_errors.middleware", level="WARNING") as logs:
|
|
137
|
+
self.trigger()
|
|
138
|
+
self.assertEqual(
|
|
139
|
+
logs.records[0].getMessage(),
|
|
140
|
+
"explain_errors: explanation truncated by OPENAI_MAX_TOKENS=321 "
|
|
141
|
+
"(reasoning tokens count against this limit; raise it if "
|
|
142
|
+
"explanations come back short or empty)",
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
def test_length_warning_unchanged_without_fallback(self):
|
|
146
|
+
self.create.return_value = _ok_response(finish_reason="length")
|
|
147
|
+
with self.assertLogs("explain_errors.middleware", level="WARNING") as logs:
|
|
148
|
+
self.trigger()
|
|
149
|
+
self.assertEqual(
|
|
150
|
+
logs.records[0].getMessage(),
|
|
151
|
+
"explain_errors: explanation truncated by OPENAI_MAX_TOKENS=321",
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
def test_empty_content_uses_failure_message_and_no_banner(self):
|
|
155
|
+
for content in (None, "", " \n"):
|
|
156
|
+
with self.subTest(content=content):
|
|
157
|
+
self.create.return_value = _ok_response(content)
|
|
158
|
+
with patch("builtins.print") as mock_print:
|
|
159
|
+
response = self.trigger()
|
|
160
|
+
self.assertEmptyContentHandled(mock_print, response)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
class AsyncTokenParamFallbackTest(TokenParamFallbackBase):
|
|
164
|
+
async def trigger(self):
|
|
165
|
+
client = getattr(self, "_client", None)
|
|
166
|
+
if client is None:
|
|
167
|
+
client = self._client = AsyncClient(raise_request_exception=False)
|
|
168
|
+
return await client.get("/boom-async/")
|
|
169
|
+
|
|
170
|
+
async def test_default_sends_max_tokens_only(self):
|
|
171
|
+
await self.trigger()
|
|
172
|
+
calls = self.sent()
|
|
173
|
+
self.assertEqual(len(calls), 1)
|
|
174
|
+
self.assertTokenParam(calls[0], "max_tokens")
|
|
175
|
+
|
|
176
|
+
async def test_rejection_retries_once_with_max_completion_tokens(self):
|
|
177
|
+
self.create.side_effect = _rejects_max_tokens
|
|
178
|
+
with self.assertLogs("explain_errors.middleware", level="INFO") as logs:
|
|
179
|
+
response = await self.trigger()
|
|
180
|
+
calls = self.sent()
|
|
181
|
+
self.assertEqual(len(calls), 2)
|
|
182
|
+
self.assertTokenParam(calls[0], "max_tokens")
|
|
183
|
+
self.assertTokenParam(calls[1], "max_completion_tokens")
|
|
184
|
+
self.assertIn("Mocked explanation.", response.content.decode())
|
|
185
|
+
self.assertEqual(len(logs.records), 1)
|
|
186
|
+
|
|
187
|
+
async def test_next_call_uses_max_completion_tokens_directly(self):
|
|
188
|
+
self.create.side_effect = _rejects_max_tokens
|
|
189
|
+
await self.trigger()
|
|
190
|
+
self.create.reset_mock()
|
|
191
|
+
await self.trigger()
|
|
192
|
+
calls = self.sent()
|
|
193
|
+
self.assertEqual(len(calls), 1)
|
|
194
|
+
self.assertTokenParam(calls[0], "max_completion_tokens")
|
|
195
|
+
|
|
196
|
+
async def test_other_param_does_not_retry(self):
|
|
197
|
+
self.create.side_effect = _bad_request("temperature", "unsupported_parameter")
|
|
198
|
+
await self.trigger()
|
|
199
|
+
self.assertEqual(len(self.sent()), 1)
|
|
200
|
+
|
|
201
|
+
async def test_other_code_does_not_retry(self):
|
|
202
|
+
self.create.side_effect = _bad_request("max_tokens", "invalid_value")
|
|
203
|
+
await self.trigger()
|
|
204
|
+
self.assertEqual(len(self.sent()), 1)
|
|
205
|
+
|
|
206
|
+
async def test_failed_retry_does_not_loop_and_uses_failure_handling(self):
|
|
207
|
+
self.create.side_effect = [
|
|
208
|
+
_bad_request("max_tokens", "unsupported_parameter"),
|
|
209
|
+
RuntimeError("api down"),
|
|
210
|
+
]
|
|
211
|
+
with patch("builtins.print") as mock_print:
|
|
212
|
+
await self.trigger()
|
|
213
|
+
self.assertEqual(len(self.sent()), 2)
|
|
214
|
+
self.assertIn(
|
|
215
|
+
"Failed to get an explanation from the model API:",
|
|
216
|
+
[call.args[0] for call in mock_print.call_args_list],
|
|
217
|
+
)
|
|
218
|
+
# The switch was not remembered after a failed retry.
|
|
219
|
+
self.create.reset_mock()
|
|
220
|
+
self.create.side_effect = None
|
|
221
|
+
self.create.return_value = _ok_response()
|
|
222
|
+
await self.trigger()
|
|
223
|
+
self.assertTokenParam(self.sent()[0], "max_tokens")
|
|
224
|
+
|
|
225
|
+
async def test_length_warning_hints_at_reasoning_tokens_after_fallback(self):
|
|
226
|
+
self.create.side_effect = _rejects_max_tokens
|
|
227
|
+
await self.trigger()
|
|
228
|
+
self.create.side_effect = None
|
|
229
|
+
self.create.return_value = _ok_response(finish_reason="length")
|
|
230
|
+
with self.assertLogs("explain_errors.middleware", level="WARNING") as logs:
|
|
231
|
+
await self.trigger()
|
|
232
|
+
self.assertIn("reasoning tokens count", logs.records[0].getMessage())
|
|
233
|
+
|
|
234
|
+
async def test_length_warning_unchanged_without_fallback(self):
|
|
235
|
+
self.create.return_value = _ok_response(finish_reason="length")
|
|
236
|
+
with self.assertLogs("explain_errors.middleware", level="WARNING") as logs:
|
|
237
|
+
await self.trigger()
|
|
238
|
+
self.assertEqual(
|
|
239
|
+
logs.records[0].getMessage(),
|
|
240
|
+
"explain_errors: explanation truncated by OPENAI_MAX_TOKENS=321",
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
async def test_empty_content_uses_failure_message_and_no_banner(self):
|
|
244
|
+
for content in (None, "", " \n"):
|
|
245
|
+
with self.subTest(content=content):
|
|
246
|
+
self.create.return_value = _ok_response(content)
|
|
247
|
+
with patch("builtins.print") as mock_print:
|
|
248
|
+
response = await self.trigger()
|
|
249
|
+
self.assertEmptyContentHandled(mock_print, response)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/management/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/rag/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/rag/retriever.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_build_error_index.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_local_embeddings.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_process_exception_split.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|