django-explain-errors 0.11.0__tar.gz → 0.11.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. {django_explain_errors-0.11.0/django_explain_errors.egg-info → django_explain_errors-0.11.1}/PKG-INFO +5 -1
  2. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/README.md +4 -0
  3. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1/django_explain_errors.egg-info}/PKG-INFO +5 -1
  4. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/django_explain_errors.egg-info/SOURCES.txt +1 -0
  5. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/middleware.py +53 -6
  6. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/setup.py +1 -1
  7. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_eval_judge.py +158 -0
  8. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_eval_run.py +183 -2
  9. django_explain_errors-0.11.1/tests/test_token_param_fallback.py +249 -0
  10. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/LICENSE +0 -0
  11. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/MANIFEST.in +0 -0
  12. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/django_explain_errors.egg-info/dependency_links.txt +0 -0
  13. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/django_explain_errors.egg-info/requires.txt +0 -0
  14. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/django_explain_errors.egg-info/top_level.txt +0 -0
  15. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/__init__.py +0 -0
  16. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/client.py +0 -0
  17. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/debug_page.py +0 -0
  18. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/management/__init__.py +0 -0
  19. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/management/commands/__init__.py +0 -0
  20. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/management/commands/build_error_index.py +0 -0
  21. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/rag/__init__.py +0 -0
  22. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/rag/indexer.py +0 -0
  23. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/rag/retriever.py +0 -0
  24. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/rag/store.py +0 -0
  25. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/sanitize.py +0 -0
  26. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/throttle.py +0 -0
  27. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/explain_errors/tracebacks.py +0 -0
  28. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/setup.cfg +0 -0
  29. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_async_marking.py +0 -0
  30. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_build_error_index.py +0 -0
  31. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_client.py +0 -0
  32. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_debug_page.py +0 -0
  33. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_dedup.py +0 -0
  34. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_eval_fixtures.py +0 -0
  35. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_language.py +0 -0
  36. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_local_embeddings.py +0 -0
  37. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_middleware.py +0 -0
  38. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_print_stdout.py +0 -0
  39. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_process_exception_split.py +0 -0
  40. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_rag.py +0 -0
  41. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_sanitize.py +0 -0
  42. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_signals.py +0 -0
  43. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_throttle.py +0 -0
  44. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_tracebacks.py +0 -0
  45. {django_explain_errors-0.11.0 → django_explain_errors-0.11.1}/tests/test_truncation.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: django-explain-errors
3
- Version: 0.11.0
3
+ Version: 0.11.1
4
4
  Summary: Django middleware that explains unhandled exceptions in DEBUG using an LLM, optionally grounded in your own project source via a local vector index. Works with OpenAI, Claude, or any OpenAI-compatible endpoint including local models.
5
5
  Home-page: https://github.com/topunix/django-explain-errors
6
6
  Author: topunix
@@ -272,6 +272,10 @@ catches the exception internally, so it never becomes an unhandled exception, bu
272
272
  than relying on `got_request_exception`. That event is unrelated to whatever error the
273
273
  developer is actually investigating.
274
274
 
275
+ OpenAI reasoning models are supported. They reject `max_tokens`, so when the API rejects it
276
+ the middleware retries once with `max_completion_tokens` (same value, `OPENAI_MAX_TOKENS`)
277
+ and keeps using that for later calls. Other providers keep receiving `max_tokens`.
278
+
275
279
  ## Configuration
276
280
 
277
281
  | Setting / variable | Required | Description |
@@ -220,6 +220,10 @@ catches the exception internally, so it never becomes an unhandled exception, bu
220
220
  than relying on `got_request_exception`. That event is unrelated to whatever error the
221
221
  developer is actually investigating.
222
222
 
223
+ OpenAI reasoning models are supported. They reject `max_tokens`, so when the API rejects it
224
+ the middleware retries once with `max_completion_tokens` (same value, `OPENAI_MAX_TOKENS`)
225
+ and keeps using that for later calls. Other providers keep receiving `max_tokens`.
226
+
223
227
  ## Configuration
224
228
 
225
229
  | Setting / variable | Required | Description |
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: django-explain-errors
3
- Version: 0.11.0
3
+ Version: 0.11.1
4
4
  Summary: Django middleware that explains unhandled exceptions in DEBUG using an LLM, optionally grounded in your own project source via a local vector index. Works with OpenAI, Claude, or any OpenAI-compatible endpoint including local models.
5
5
  Home-page: https://github.com/topunix/django-explain-errors
6
6
  Author: topunix
@@ -272,6 +272,10 @@ catches the exception internally, so it never becomes an unhandled exception, bu
272
272
  than relying on `got_request_exception`. That event is unrelated to whatever error the
273
273
  developer is actually investigating.
274
274
 
275
+ OpenAI reasoning models are supported. They reject `max_tokens`, so when the API rejects it
276
+ the middleware retries once with `max_completion_tokens` (same value, `OPENAI_MAX_TOKENS`)
277
+ and keeps using that for later calls. Other providers keep receiving `max_tokens`.
278
+
275
279
  ## Configuration
276
280
 
277
281
  | Setting / variable | Required | Description |
@@ -38,5 +38,6 @@ tests/test_rag.py
38
38
  tests/test_sanitize.py
39
39
  tests/test_signals.py
40
40
  tests/test_throttle.py
41
+ tests/test_token_param_fallback.py
41
42
  tests/test_tracebacks.py
42
43
  tests/test_truncation.py
@@ -3,6 +3,7 @@ import logging
3
3
  import threading
4
4
  from collections import OrderedDict
5
5
 
6
+ import openai
6
7
  from django.conf import settings
7
8
  from django.http import JsonResponse
8
9
  from asgiref.sync import markcoroutinefunction, sync_to_async
@@ -81,6 +82,9 @@ class ExplainErrorsMiddleware:
81
82
  # above this one runs in sync mode and receives a coroutine.
82
83
  markcoroutinefunction(self)
83
84
  self.openai_client = None
85
+ # Output-cap parameter sent to the model. Reasoning models reject
86
+ # max_tokens; _create_completion switches this once, per instance.
87
+ self._token_param = "max_tokens"
84
88
  self.throttle = None
85
89
  self.dedup_enabled = False
86
90
  self._explanation_cache = OrderedDict()
@@ -251,6 +255,39 @@ class ExplainErrorsMiddleware:
251
255
 
252
256
  return prompt
253
257
 
258
+ def _create_completion(self, messages):
259
+ """Send the chat completion, adapting to the model's output-cap parameter.
260
+
261
+ Sends max_tokens by default. If the API rejects exactly that parameter
262
+ (OpenAI reasoning models require max_completion_tokens), retries once
263
+ with max_completion_tokens and remembers the choice on this instance.
264
+ Any other error, or a failed retry, propagates to the caller.
265
+ """
266
+
267
+ def create(token_param):
268
+ return self.openai_client.chat.completions.create(
269
+ model=self.model,
270
+ messages=messages,
271
+ **{token_param: self.max_tokens},
272
+ )
273
+
274
+ try:
275
+ return create(self._token_param)
276
+ except openai.BadRequestError as exc:
277
+ if self._token_param != "max_tokens" or not (
278
+ getattr(exc, "param", None) == "max_tokens"
279
+ and getattr(exc, "code", None) == "unsupported_parameter"
280
+ ):
281
+ raise
282
+ response = create("max_completion_tokens")
283
+ self._token_param = "max_completion_tokens"
284
+ logger.info(
285
+ "explain_errors: model %s rejected max_tokens; using "
286
+ "max_completion_tokens instead",
287
+ self.model,
288
+ )
289
+ return response
290
+
254
291
  def _generate_explanation(self, prompt):
255
292
  """Call the model and return the explanation text, or None on failure.
256
293
 
@@ -260,13 +297,11 @@ class ExplainErrorsMiddleware:
260
297
  explanation = None
261
298
  try:
262
299
  # Call model API
263
- response = self.openai_client.chat.completions.create(
264
- model=self.model,
265
- messages=[
300
+ response = self._create_completion(
301
+ [
266
302
  {"role": "system", "content": self.system_prompt},
267
303
  {"role": "user", "content": prompt},
268
- ],
269
- max_tokens=self.max_tokens,
304
+ ]
270
305
  )
271
306
  explanation = response.choices[0].message.content
272
307
  usage = getattr(response, "usage", None)
@@ -276,11 +311,23 @@ class ExplainErrorsMiddleware:
276
311
  getattr(usage, "completion_tokens", None),
277
312
  )
278
313
  if response.choices[0].finish_reason == "length":
314
+ hint = ""
315
+ if self._token_param == "max_completion_tokens":
316
+ hint = (
317
+ " (reasoning tokens count against this limit; raise it "
318
+ "if explanations come back short or empty)"
319
+ )
279
320
  logger.warning(
280
- "explain_errors: explanation truncated by OPENAI_MAX_TOKENS=%s",
321
+ "explain_errors: explanation truncated by OPENAI_MAX_TOKENS=%s%s",
281
322
  self.max_tokens,
323
+ hint,
282
324
  )
283
325
 
326
+ if not explanation or not explanation.strip():
327
+ # Reasoning can consume the whole budget and leave no text.
328
+ explanation = None
329
+ raise ValueError("the model returned an empty response")
330
+
284
331
  if getattr(settings, "EXPLAIN_ERRORS_PRINT_STDOUT", True):
285
332
  print(f"Error explanation ({self.model}):\n", explanation)
286
333
  except Exception as e:
@@ -10,7 +10,7 @@ with open('README.md', encoding='utf-8') as f:
10
10
 
11
11
  setup(
12
12
  name='django-explain-errors',
13
- version='0.11.0',
13
+ version='0.11.1',
14
14
  packages=find_packages(exclude=['tests', 'tests.*', 'evals', 'evals.*']),
15
15
  description='Django middleware that explains unhandled exceptions in DEBUG using an LLM, optionally grounded in your own project source via a local vector index. Works with OpenAI, Claude, or any OpenAI-compatible endpoint including local models.',
16
16
  long_description_content_type='text/markdown',
@@ -391,3 +391,161 @@ class CompareClaimsGuardTest(unittest.TestCase):
391
391
  )
392
392
 
393
393
  self.assertFalse(result["judge_failure"])
394
+
395
+
396
+ def _openai_error(cls, status=None):
397
+ """An instance of the openai exception class without constructing its
398
+ httpx request/response (which httpx package openai imports varies by
399
+ openai version).
400
+ """
401
+ exc = Exception.__new__(cls)
402
+ exc.args = ("boom",)
403
+ if status is not None:
404
+ exc.status_code = status
405
+ return exc
406
+
407
+
408
+ def _flaky_client(errors, then=VALID_RESPONSE):
409
+ """Raises each of `errors` in turn, then returns `then`."""
410
+ client = MagicMock()
411
+ ok = MagicMock(
412
+ choices=[MagicMock(message=MagicMock(content=then))],
413
+ usage=MagicMock(prompt_tokens=7, completion_tokens=3),
414
+ )
415
+ client.chat.completions.create.side_effect = list(errors) + [ok]
416
+ return client
417
+
418
+
419
+ class CompareRetryTest(unittest.TestCase):
420
+
421
+ def test_pre_drawn_position_is_used_and_rng_untouched(self):
422
+ fixture = _make_fixture()
423
+ client = _client_returning(VALID_RESPONSE)
424
+
425
+ class ExplodingRng:
426
+ def random(self):
427
+ raise AssertionError("rng must not be drawn when rag_on_is_a is given")
428
+
429
+ result = compare(
430
+ client, "m", "tb", fixture, "src", "off", "on",
431
+ rng=ExplodingRng(), rag_on_is_a=False,
432
+ )
433
+
434
+ self.assertFalse(result["rag_on_is_a"])
435
+
436
+ def test_rate_limit_is_retried_and_counted(self):
437
+ import openai
438
+
439
+ fixture = _make_fixture()
440
+ errors = [_openai_error(openai.RateLimitError, 429)] * 2
441
+ client = _flaky_client(errors)
442
+ sleeps = []
443
+
444
+ result = compare(
445
+ client, "m", "tb", fixture, "src", "off", "on",
446
+ rng=_FixedRng(0.1), sleep=sleeps.append,
447
+ )
448
+
449
+ self.assertFalse(result["judge_failure"])
450
+ self.assertEqual(result["retries"], 2)
451
+ self.assertEqual(len(sleeps), 2)
452
+ self.assertEqual(client.chat.completions.create.call_count, 3)
453
+ self.assertEqual(result["prompt_tokens"], 7)
454
+
455
+ def test_backoff_grows_and_is_capped(self):
456
+ import openai
457
+
458
+ from evals import judge
459
+
460
+ fixture = _make_fixture()
461
+ client = _flaky_client([_openai_error(openai.InternalServerError, 503)] * 4)
462
+ ceilings = []
463
+
464
+ class RecordingJitter:
465
+ def uniform(self, low, high):
466
+ ceilings.append(high)
467
+ return high
468
+
469
+ original = judge._jitter_rng
470
+ judge._jitter_rng = RecordingJitter()
471
+ self.addCleanup(setattr, judge, "_jitter_rng", original)
472
+
473
+ compare(
474
+ client, "m", "tb", fixture, "src", "off", "on",
475
+ rng=_FixedRng(0.1), max_attempts=5, sleep=lambda s: None,
476
+ )
477
+
478
+ self.assertEqual(ceilings, [1.0, 2.0, 4.0, 8.0])
479
+
480
+ def test_timeout_and_connection_errors_are_retried(self):
481
+ import openai
482
+
483
+ fixture = _make_fixture()
484
+ client = _flaky_client(
485
+ [_openai_error(openai.APITimeoutError), _openai_error(openai.APIConnectionError)]
486
+ )
487
+
488
+ result = compare(
489
+ client, "m", "tb", fixture, "src", "off", "on",
490
+ rng=_FixedRng(0.1), sleep=lambda s: None,
491
+ )
492
+
493
+ self.assertFalse(result["judge_failure"])
494
+ self.assertEqual(result["retries"], 2)
495
+
496
+ def test_exhausted_retries_is_a_judge_failure_with_retries_recorded(self):
497
+ import openai
498
+
499
+ fixture = _make_fixture()
500
+ client = MagicMock()
501
+ client.chat.completions.create.side_effect = _openai_error(openai.RateLimitError, 429)
502
+
503
+ result = compare(
504
+ client, "m", "tb", fixture, "src", "off", "on",
505
+ rng=_FixedRng(0.1), max_attempts=3, sleep=lambda s: None,
506
+ )
507
+
508
+ self.assertTrue(result["judge_failure"])
509
+ self.assertEqual(result["retries"], 2)
510
+ self.assertEqual(client.chat.completions.create.call_count, 3)
511
+ self.assertIn("boom", result["error"])
512
+ self.assertIn("rag_on_is_a", result)
513
+
514
+ def test_non_transient_error_is_not_retried(self):
515
+ import openai
516
+
517
+ fixture = _make_fixture()
518
+ client = MagicMock()
519
+ client.chat.completions.create.side_effect = _openai_error(
520
+ openai.AuthenticationError, 401
521
+ )
522
+
523
+ result = compare(
524
+ client, "m", "tb", fixture, "src", "off", "on",
525
+ rng=_FixedRng(0.1), sleep=lambda s: self.fail("must not sleep"),
526
+ )
527
+
528
+ self.assertTrue(result["judge_failure"])
529
+ self.assertEqual(result["retries"], 0)
530
+ self.assertEqual(client.chat.completions.create.call_count, 1)
531
+
532
+ def test_malformed_response_is_not_retried(self):
533
+ fixture = _make_fixture()
534
+ client = _client_returning("not json")
535
+
536
+ result = compare(
537
+ client, "m", "tb", fixture, "src", "off", "on",
538
+ rng=_FixedRng(0.1), sleep=lambda s: self.fail("must not sleep"),
539
+ )
540
+
541
+ self.assertTrue(result["judge_failure"])
542
+ self.assertEqual(result["retries"], 0)
543
+ self.assertEqual(client.chat.completions.create.call_count, 1)
544
+
545
+ def test_clean_call_records_zero_retries(self):
546
+ fixture = _make_fixture()
547
+ client = _client_returning(VALID_RESPONSE)
548
+
549
+ result = compare(client, "m", "tb", fixture, "src", "off", "on", rng=_FixedRng(0.1))
550
+
551
+ self.assertEqual(result["retries"], 0)
@@ -8,9 +8,22 @@ import json
8
8
  import os
9
9
  import subprocess
10
10
  import sys
11
+ import random
12
+ import runpy
13
+ import threading
14
+ import time
11
15
  import unittest
12
-
13
- from evals.run import estimate_cost_usd, tally_judgments, tally_latency, tally_usage
16
+ from unittest.mock import patch
17
+
18
+ from evals.fixtures import FIXTURES_BY_NAME
19
+ from evals.run import (
20
+ estimate_cost_usd,
21
+ judge_all,
22
+ tally_judgments,
23
+ tally_latency,
24
+ tally_retries,
25
+ tally_usage,
26
+ )
14
27
 
15
28
  REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
16
29
 
@@ -197,6 +210,11 @@ class TallyLatencyAndUsageTest(unittest.TestCase):
197
210
  cost = estimate_cost_usd("gpt-4o-mini", 1000, 1000)
198
211
  self.assertAlmostEqual(cost, 0.00015 + 0.0006)
199
212
 
213
+ def test_estimate_cost_for_gpt_6_luna(self):
214
+ cost = estimate_cost_usd("gpt-6-luna", 1000, 1000)
215
+ self.assertIsNotNone(cost)
216
+ self.assertAlmostEqual(cost, 0.0001 + 0.0005)
217
+
200
218
 
201
219
  class BuildJudgeSourceTest(unittest.TestCase):
202
220
  """Runs tests/_eval_judge_source_helper.py in a subprocess -- see its
@@ -279,6 +297,161 @@ class BuildJudgeSourceTest(unittest.TestCase):
279
297
  self.assertFalse(any(label.endswith(".html") for label in labels))
280
298
 
281
299
 
300
+ class JudgeAllConcurrencyTest(unittest.TestCase):
301
+ """judge_all with a fake judge: position draws happen up front, in
302
+ order, independent of concurrency, and results keep job order.
303
+ """
304
+
305
+ FIXTURE_NAMES = ("none_attribute", "str_recursion", "bad_lookup")
306
+ RUNS = 3
307
+ SEED = 20260711
308
+
309
+ def _calls(self):
310
+ return [
311
+ {
312
+ "fixture": name,
313
+ "rag_enabled": rag,
314
+ "run_index": run,
315
+ "traceback": "tb",
316
+ "explanation": f"{name}/{rag}/{run}",
317
+ }
318
+ for name in self.FIXTURE_NAMES
319
+ for rag in (False, True)
320
+ for run in range(self.RUNS)
321
+ ]
322
+
323
+ def _run(self, concurrency, scramble=True):
324
+ """Returns (judgments, draws_before_first_call, call_order)."""
325
+ fixtures = [FIXTURES_BY_NAME[n] for n in self.FIXTURE_NAMES]
326
+ draws = []
327
+ lock = threading.Lock()
328
+ state = {"draws_at_first_call": None}
329
+ call_order = []
330
+
331
+ class CountingRandom(random.Random):
332
+ def random(self):
333
+ value = super().random()
334
+ draws.append(value)
335
+ return value
336
+
337
+ def fake_compare(client, model, tb, fixture, source, off, on, rag_on_is_a=None, **kw):
338
+ with lock:
339
+ if state["draws_at_first_call"] is None:
340
+ state["draws_at_first_call"] = len(draws)
341
+ call_order.append((fixture.name, off))
342
+ if scramble:
343
+ # Finish out of submission order: later jobs return sooner.
344
+ time.sleep(0.02 if off.endswith("/0") else 0.0)
345
+ return {
346
+ "fixture": fixture.name,
347
+ "run_index": None,
348
+ "rag_on_is_a": rag_on_is_a,
349
+ "off": off,
350
+ "on": on,
351
+ "retries": 0,
352
+ }
353
+
354
+ with patch("evals.judge.get_judge_client", return_value=object()), patch(
355
+ "evals.judge.get_judge_model", return_value="m"
356
+ ), patch("evals.judge.compare", side_effect=fake_compare), patch(
357
+ "evals.run._build_judge_source", return_value=[]
358
+ ), patch("evals.run.random.Random", CountingRandom):
359
+ judgments = judge_all(
360
+ fixtures, self._calls(), {}, concurrency=concurrency, seed=self.SEED
361
+ )
362
+ return judgments, state["draws_at_first_call"], call_order
363
+
364
+ def test_all_draws_happen_before_the_first_judge_call(self):
365
+ total = len(self.FIXTURE_NAMES) * self.RUNS
366
+ for n in (1, 4, 8):
367
+ _, draws_at_first_call, _ = self._run(n)
368
+ self.assertEqual(draws_at_first_call, total, f"N={n}")
369
+
370
+ def test_positions_match_the_seeded_rng_in_fixture_then_run_order(self):
371
+ reference = random.Random(self.SEED)
372
+ expected = [
373
+ reference.random() < 0.5
374
+ for _ in range(len(self.FIXTURE_NAMES) * self.RUNS)
375
+ ]
376
+ for n in (1, 2, 8, 32):
377
+ judgments, _, _ = self._run(n)
378
+ self.assertEqual([j["rag_on_is_a"] for j in judgments], expected, f"N={n}")
379
+
380
+ def test_results_keep_comparison_order_even_when_completion_order_differs(self):
381
+ expected_order = [
382
+ (name, run) for name in self.FIXTURE_NAMES for run in range(self.RUNS)
383
+ ]
384
+ for n in (1, 8):
385
+ judgments, _, _ = self._run(n)
386
+ self.assertEqual(
387
+ [(j["fixture"], j["run_index"]) for j in judgments], expected_order, f"N={n}"
388
+ )
389
+ self.assertEqual(
390
+ [j["off"] for j in judgments],
391
+ [f"{name}/False/{run}" for name, run in expected_order],
392
+ )
393
+
394
+ def test_concurrency_one_calls_in_order(self):
395
+ _, _, call_order = self._run(1)
396
+ self.assertEqual(
397
+ [name for name, _ in call_order],
398
+ [name for name in self.FIXTURE_NAMES for _ in range(self.RUNS)],
399
+ )
400
+
401
+ def test_concurrency_actually_overlaps_calls(self):
402
+ active = {"now": 0, "peak": 0}
403
+ lock = threading.Lock()
404
+
405
+ def fake_compare(*args, **kwargs):
406
+ with lock:
407
+ active["now"] += 1
408
+ active["peak"] = max(active["peak"], active["now"])
409
+ time.sleep(0.05)
410
+ with lock:
411
+ active["now"] -= 1
412
+ return {"fixture": "x", "retries": 0}
413
+
414
+ fixtures = [FIXTURES_BY_NAME[n] for n in self.FIXTURE_NAMES]
415
+ with patch("evals.judge.get_judge_client", return_value=object()), patch(
416
+ "evals.judge.get_judge_model", return_value="m"
417
+ ), patch("evals.judge.compare", side_effect=fake_compare), patch(
418
+ "evals.run._build_judge_source", return_value=[]
419
+ ):
420
+ judge_all(fixtures, self._calls(), {}, concurrency=4, seed=1)
421
+ self.assertGreater(active["peak"], 1)
422
+ self.assertLessEqual(active["peak"], 4)
423
+
424
+
425
+ class TallyRetriesTest(unittest.TestCase):
426
+
427
+ def test_sums_per_judgment_retries_and_tolerates_missing(self):
428
+ self.assertEqual(
429
+ tally_retries([{"retries": 2}, {"retries": 0}, {}, {"retries": None}, {"retries": 1}]),
430
+ 3,
431
+ )
432
+
433
+
434
+ class FixtureSettingsModelTest(unittest.TestCase):
435
+ """The fixture app's settings resolve OPENAI_MODEL from the environment."""
436
+
437
+ SETTINGS_PATH = os.path.join(REPO_ROOT, "evals", "fixture_app", "settings.py")
438
+
439
+ def _model(self, **env):
440
+ with patch.dict(os.environ, env):
441
+ if "OPENAI_MODEL" not in env:
442
+ os.environ.pop("OPENAI_MODEL", None)
443
+ return runpy.run_path(self.SETTINGS_PATH)["OPENAI_MODEL"]
444
+
445
+ def test_reads_env_var(self):
446
+ self.assertEqual(self._model(OPENAI_MODEL="some-other-model"), "some-other-model")
447
+
448
+ def test_unset_falls_back_to_default(self):
449
+ self.assertEqual(self._model(), "gpt-4o-mini")
450
+
451
+ def test_empty_string_falls_back_to_default(self):
452
+ self.assertEqual(self._model(OPENAI_MODEL=""), "gpt-4o-mini")
453
+
454
+
282
455
  class EvalHarnessEndToEndTest(unittest.TestCase):
283
456
  """Runs evals.run.main() in a subprocess with the generator (chat +
284
457
  embeddings) and the judge fully mocked
@@ -326,10 +499,17 @@ class EvalHarnessEndToEndTest(unittest.TestCase):
326
499
  "generator_model",
327
500
  "judge_model",
328
501
  "runs",
502
+ "seed",
503
+ "timing",
504
+ "judge_retries_total",
329
505
  "calls",
330
506
  "judgments",
331
507
  ):
332
508
  self.assertIn(key, data)
509
+ self.assertEqual(data["judge_retries_total"], 0)
510
+ self.assertEqual(data["timing"]["judge_concurrency"], 8)
511
+ self.assertGreaterEqual(data["timing"]["generator_seconds"], 0)
512
+ self.assertGreaterEqual(data["timing"]["judge_seconds"], 0)
333
513
 
334
514
  self.assertEqual(data["generator_model"], "gpt-4o-mini")
335
515
  self.assertEqual(data["judge_model"], "judge-test-model")
@@ -369,6 +549,7 @@ class EvalHarnessEndToEndTest(unittest.TestCase):
369
549
  "b",
370
550
  "reasoning",
371
551
  "run_index",
552
+ "retries",
372
553
  }
373
554
  for judgment in data["judgments"]:
374
555
  self.assertTrue(judgment_keys.issubset(judgment.keys()), judgment)
@@ -0,0 +1,249 @@
1
+ from unittest.mock import MagicMock, patch
2
+
3
+ import openai
4
+ from django.test import AsyncClient, Client, SimpleTestCase, override_settings
5
+
6
+
7
+ def _bad_request(param, code):
8
+ body = {
9
+ "message": "Unsupported parameter",
10
+ "type": "invalid_request_error",
11
+ "param": param,
12
+ "code": code,
13
+ }
14
+ response = MagicMock(status_code=400)
15
+ return openai.BadRequestError("Unsupported parameter", response=response, body=body)
16
+
17
+
18
+ def _ok_response(text="Mocked explanation.", finish_reason="stop"):
19
+ return MagicMock(
20
+ choices=[
21
+ MagicMock(message=MagicMock(content=text), finish_reason=finish_reason)
22
+ ]
23
+ )
24
+
25
+
26
+ def _rejects_max_tokens(**kwargs):
27
+ if "max_tokens" in kwargs:
28
+ raise _bad_request("max_tokens", "unsupported_parameter")
29
+ return _ok_response()
30
+
31
+
32
+ @override_settings(
33
+ DEBUG=True,
34
+ OPENAI_API_KEY="test-key",
35
+ OPENAI_MAX_TOKENS=321,
36
+ EXPLAIN_ERRORS_DEDUP=False,
37
+ ROOT_URLCONF="tests.urls_debug_page",
38
+ MIDDLEWARE=["explain_errors.middleware.ExplainErrorsMiddleware"],
39
+ )
40
+ class TokenParamFallbackBase(SimpleTestCase):
41
+ def setUp(self):
42
+ patcher = patch("openai.OpenAI")
43
+ mock_cls = patcher.start()
44
+ self.addCleanup(patcher.stop)
45
+ self.create = MagicMock(return_value=_ok_response())
46
+ mock_cls.return_value.chat.completions.create = self.create
47
+
48
+ def sent(self):
49
+ return [call.kwargs for call in self.create.call_args_list]
50
+
51
+ def assertTokenParam(self, kwargs, expected):
52
+ other = (
53
+ "max_completion_tokens" if expected == "max_tokens" else "max_tokens"
54
+ )
55
+ self.assertEqual(kwargs[expected], 321)
56
+ self.assertNotIn(other, kwargs)
57
+
58
+ def assertEmptyContentHandled(self, mock_print, response):
59
+ firsts = [call.args[0] for call in mock_print.call_args_list if call.args]
60
+ self.assertEqual(
61
+ firsts.count("Failed to get an explanation from the model API:"), 1
62
+ )
63
+ self.assertFalse([f for f in firsts if str(f).startswith("Error explanation")])
64
+ self.assertNotIn('id="explain-errors"', response.content.decode())
65
+
66
+
67
+ class SyncTokenParamFallbackTest(TokenParamFallbackBase):
68
+ def trigger(self):
69
+ client = getattr(self, "_client", None)
70
+ if client is None:
71
+ client = self._client = Client(raise_request_exception=False)
72
+ return client.get("/boom/")
73
+
74
+ def test_default_sends_max_tokens_only(self):
75
+ self.trigger()
76
+ calls = self.sent()
77
+ self.assertEqual(len(calls), 1)
78
+ self.assertTokenParam(calls[0], "max_tokens")
79
+
80
+ def test_rejection_retries_once_with_max_completion_tokens(self):
81
+ self.create.side_effect = _rejects_max_tokens
82
+ with self.assertLogs("explain_errors.middleware", level="INFO") as logs:
83
+ response = self.trigger()
84
+ calls = self.sent()
85
+ self.assertEqual(len(calls), 2)
86
+ self.assertTokenParam(calls[0], "max_tokens")
87
+ self.assertTokenParam(calls[1], "max_completion_tokens")
88
+ self.assertIn("Mocked explanation.", response.content.decode())
89
+ self.assertEqual(len(logs.records), 1)
90
+ self.assertIn("max_tokens", logs.records[0].getMessage())
91
+ self.assertIn("max_completion_tokens", logs.records[0].getMessage())
92
+
93
+ def test_next_call_uses_max_completion_tokens_directly(self):
94
+ self.create.side_effect = _rejects_max_tokens
95
+ self.trigger()
96
+ self.create.reset_mock()
97
+ self.trigger()
98
+ calls = self.sent()
99
+ self.assertEqual(len(calls), 1)
100
+ self.assertTokenParam(calls[0], "max_completion_tokens")
101
+
102
+ def test_other_param_does_not_retry(self):
103
+ self.create.side_effect = _bad_request("temperature", "unsupported_parameter")
104
+ self.trigger()
105
+ self.assertEqual(len(self.sent()), 1)
106
+
107
+ def test_other_code_does_not_retry(self):
108
+ self.create.side_effect = _bad_request("max_tokens", "invalid_value")
109
+ self.trigger()
110
+ self.assertEqual(len(self.sent()), 1)
111
+
112
+ def test_failed_retry_does_not_loop_and_uses_failure_handling(self):
113
+ self.create.side_effect = [
114
+ _bad_request("max_tokens", "unsupported_parameter"),
115
+ RuntimeError("api down"),
116
+ ]
117
+ with patch("builtins.print") as mock_print:
118
+ self.trigger()
119
+ self.assertEqual(len(self.sent()), 2)
120
+ self.assertIn(
121
+ "Failed to get an explanation from the model API:",
122
+ [call.args[0] for call in mock_print.call_args_list],
123
+ )
124
+ # The switch was not remembered after a failed retry.
125
+ self.create.reset_mock()
126
+ self.create.side_effect = None
127
+ self.create.return_value = _ok_response()
128
+ self.trigger()
129
+ self.assertTokenParam(self.sent()[0], "max_tokens")
130
+
131
+ def test_length_warning_hints_at_reasoning_tokens_after_fallback(self):
132
+ self.create.side_effect = _rejects_max_tokens
133
+ self.trigger()
134
+ self.create.side_effect = None
135
+ self.create.return_value = _ok_response(finish_reason="length")
136
+ with self.assertLogs("explain_errors.middleware", level="WARNING") as logs:
137
+ self.trigger()
138
+ self.assertEqual(
139
+ logs.records[0].getMessage(),
140
+ "explain_errors: explanation truncated by OPENAI_MAX_TOKENS=321 "
141
+ "(reasoning tokens count against this limit; raise it if "
142
+ "explanations come back short or empty)",
143
+ )
144
+
145
+ def test_length_warning_unchanged_without_fallback(self):
146
+ self.create.return_value = _ok_response(finish_reason="length")
147
+ with self.assertLogs("explain_errors.middleware", level="WARNING") as logs:
148
+ self.trigger()
149
+ self.assertEqual(
150
+ logs.records[0].getMessage(),
151
+ "explain_errors: explanation truncated by OPENAI_MAX_TOKENS=321",
152
+ )
153
+
154
+ def test_empty_content_uses_failure_message_and_no_banner(self):
155
+ for content in (None, "", " \n"):
156
+ with self.subTest(content=content):
157
+ self.create.return_value = _ok_response(content)
158
+ with patch("builtins.print") as mock_print:
159
+ response = self.trigger()
160
+ self.assertEmptyContentHandled(mock_print, response)
161
+
162
+
163
+ class AsyncTokenParamFallbackTest(TokenParamFallbackBase):
164
+ async def trigger(self):
165
+ client = getattr(self, "_client", None)
166
+ if client is None:
167
+ client = self._client = AsyncClient(raise_request_exception=False)
168
+ return await client.get("/boom-async/")
169
+
170
+ async def test_default_sends_max_tokens_only(self):
171
+ await self.trigger()
172
+ calls = self.sent()
173
+ self.assertEqual(len(calls), 1)
174
+ self.assertTokenParam(calls[0], "max_tokens")
175
+
176
+ async def test_rejection_retries_once_with_max_completion_tokens(self):
177
+ self.create.side_effect = _rejects_max_tokens
178
+ with self.assertLogs("explain_errors.middleware", level="INFO") as logs:
179
+ response = await self.trigger()
180
+ calls = self.sent()
181
+ self.assertEqual(len(calls), 2)
182
+ self.assertTokenParam(calls[0], "max_tokens")
183
+ self.assertTokenParam(calls[1], "max_completion_tokens")
184
+ self.assertIn("Mocked explanation.", response.content.decode())
185
+ self.assertEqual(len(logs.records), 1)
186
+
187
+ async def test_next_call_uses_max_completion_tokens_directly(self):
188
+ self.create.side_effect = _rejects_max_tokens
189
+ await self.trigger()
190
+ self.create.reset_mock()
191
+ await self.trigger()
192
+ calls = self.sent()
193
+ self.assertEqual(len(calls), 1)
194
+ self.assertTokenParam(calls[0], "max_completion_tokens")
195
+
196
+ async def test_other_param_does_not_retry(self):
197
+ self.create.side_effect = _bad_request("temperature", "unsupported_parameter")
198
+ await self.trigger()
199
+ self.assertEqual(len(self.sent()), 1)
200
+
201
+ async def test_other_code_does_not_retry(self):
202
+ self.create.side_effect = _bad_request("max_tokens", "invalid_value")
203
+ await self.trigger()
204
+ self.assertEqual(len(self.sent()), 1)
205
+
206
+ async def test_failed_retry_does_not_loop_and_uses_failure_handling(self):
207
+ self.create.side_effect = [
208
+ _bad_request("max_tokens", "unsupported_parameter"),
209
+ RuntimeError("api down"),
210
+ ]
211
+ with patch("builtins.print") as mock_print:
212
+ await self.trigger()
213
+ self.assertEqual(len(self.sent()), 2)
214
+ self.assertIn(
215
+ "Failed to get an explanation from the model API:",
216
+ [call.args[0] for call in mock_print.call_args_list],
217
+ )
218
+ # The switch was not remembered after a failed retry.
219
+ self.create.reset_mock()
220
+ self.create.side_effect = None
221
+ self.create.return_value = _ok_response()
222
+ await self.trigger()
223
+ self.assertTokenParam(self.sent()[0], "max_tokens")
224
+
225
+ async def test_length_warning_hints_at_reasoning_tokens_after_fallback(self):
226
+ self.create.side_effect = _rejects_max_tokens
227
+ await self.trigger()
228
+ self.create.side_effect = None
229
+ self.create.return_value = _ok_response(finish_reason="length")
230
+ with self.assertLogs("explain_errors.middleware", level="WARNING") as logs:
231
+ await self.trigger()
232
+ self.assertIn("reasoning tokens count", logs.records[0].getMessage())
233
+
234
+ async def test_length_warning_unchanged_without_fallback(self):
235
+ self.create.return_value = _ok_response(finish_reason="length")
236
+ with self.assertLogs("explain_errors.middleware", level="WARNING") as logs:
237
+ await self.trigger()
238
+ self.assertEqual(
239
+ logs.records[0].getMessage(),
240
+ "explain_errors: explanation truncated by OPENAI_MAX_TOKENS=321",
241
+ )
242
+
243
+ async def test_empty_content_uses_failure_message_and_no_banner(self):
244
+ for content in (None, "", " \n"):
245
+ with self.subTest(content=content):
246
+ self.create.return_value = _ok_response(content)
247
+ with patch("builtins.print") as mock_print:
248
+ response = await self.trigger()
249
+ self.assertEmptyContentHandled(mock_print, response)