langfun 0.0.2.dev20240330__py3-none-any.whl → 0.1.2.dev202501140804__py3-none-any.whl

langfun/core/llms/anthropic_test.py

@@ -0,0 +1,235 @@
+# Copyright 2023 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for Anthropic models."""
+
+import base64
+import os
+from typing import Any
+import unittest
+from unittest import mock
+
+from google.auth import exceptions
+from langfun.core import language_model
+from langfun.core import message as lf_message
+from langfun.core import modalities as lf_modalities
+from langfun.core.llms import anthropic
+import pyglove as pg
+import requests
+
+
+def mock_requests_post(url: str, json: dict[str, Any], **kwargs):
+  del url, kwargs
+
+  response = requests.Response()
+  response.status_code = 200
+  response._content = pg.to_json_str({
+      'content': [{
+          'type': 'text',
+          'text': (
+              f'hello with temperature={json.get("temperature")}, '
+              f'top_k={json.get("top_k")}, '
+              f'top_p={json.get("top_p")}, '
+              f'max_tokens={json.get("max_tokens")}, '
+              f'stop={json.get("stop_sequences")}.'
+          ),
+      }],
+      'usage': {
+          'input_tokens': 2,
+          'output_tokens': 1,
+      },
+  }).encode()
+  return response
+
+
+image_content = (
+    b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x18\x00\x00\x00\x18\x04'
+    b'\x03\x00\x00\x00\x12Y \xcb\x00\x00\x00\x18PLTE\x00\x00'
+    b'\x00fff_chaag_cg_ch^ci_ciC\xedb\x94\x00\x00\x00\x08tRNS'
+    b'\x00\n\x9f*\xd4\xff_\xf4\xe4\x8b\xf3a\x00\x00\x00>IDATx'
+    b'\x01c \x05\x08)"\xd8\xcc\xae!\x06pNz\x88k\x19\\Q\xa8"\x10'
+    b'\xc1\x14\x95\x01%\xc1\n\xa143Ta\xa8"D-\x84\x03QM\x98\xc3'
+    b'\x1a\x1a\x1a@5\x0e\x04\xa0q\x88\x05\x00\x07\xf8\x18\xf9'
+    b'\xdao\xd0|\x00\x00\x00\x00IEND\xaeB`\x82'
+)
+
+pdf_content = (
+    b'%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<<'
+    b' /Type /Pages /Count 1 /Kids [3 0 R] >>\nendobj\n3 0 obj\n<< /Type /Page'
+    b' /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R >>\nendobj\n4 0'
+    b' obj\n<< /Length 44 >>\nstream\nBT /F1 24 Tf 100 700 Td (Hello, PDF'
+    b' content!) Tj ET\nendstream\nendobj\n5 0 obj\n<< /Type /Font /Subtype'
+    b' /Type1 /BaseFont /Helvetica >>\nendobj\nxref\n0 6\n0000000000 65535 f'
+    b' \n0000000010 00000 n \n0000000079 00000 n \n0000000178 00000 n'
+    b' \n0000000278 00000 n \n0000000407 00000 n \ntrailer\n<< /Size 6 /Root 1'
+    b' 0 R >>\nstartxref\n517\n%%EOF'
+)
+
+
+def mock_mm_requests_post(url: str, json: dict[str, Any], **kwargs):
+  del url, kwargs
+  v = json['messages'][0]['content'][0]
+  content = lf_modalities.Mime.from_bytes(base64.b64decode(v['source']['data']))
+
+  response = requests.Response()
+  response.status_code = 200
+  response._content = pg.to_json_str({
+      'content': [{
+          'type': 'text',
+          'text': f'{v["type"]}: {content.mime_type}',
+      }],
+      'usage': {
+          'input_tokens': 2,
+          'output_tokens': 1,
+      },
+  }).encode()
+  return response
+
+
+def mock_requests_post_error(status_code, error_type, error_message):
+  def _mock_requests(url: str, json: dict[str, Any], **kwargs):
+    del url, json, kwargs
+    response = requests.Response()
+    response.status_code = status_code
+    response._content = pg.to_json_str(
+        {
+            'error': {
+                'type': error_type,
+                'message': error_message,
+            }
+        }
+    ).encode()
+    return response
+
+  return _mock_requests
+
+
+class AnthropicTest(unittest.TestCase):
+
+  def test_basics(self):
+    self.assertEqual(
+        anthropic.Claude3Haiku().model_id, 'claude-3-haiku-20240307'
+    )
+    self.assertGreater(anthropic.Claude3Haiku().max_concurrency, 0)
+
+  def test_api_key(self):
+    lm = anthropic.Claude3Haiku()
+    with self.assertRaisesRegex(ValueError, 'Please specify `api_key`'):
+      lm('hi')
+
+    with mock.patch('requests.Session.post') as mock_request:
+      mock_request.side_effect = mock_requests_post
+
+      lm = anthropic.Claude3Haiku(api_key='fake key')
+      self.assertRegex(lm('hi').text, 'hello.*')
+
+      os.environ['ANTHROPIC_API_KEY'] = 'abc'
+      lm = anthropic.Claude3Haiku()
+      self.assertRegex(lm('hi').text, 'hello.*')
+      del os.environ['ANTHROPIC_API_KEY']
+
+  def test_call(self):
+    with mock.patch('requests.Session.post') as mock_request:
+      mock_request.side_effect = mock_requests_post
+      lm = anthropic.Claude3Haiku(api_key='fake_key')
+      response = lm('hello', temperature=0.0, top_k=0.1, top_p=0.2, stop=['\n'])
+      self.assertEqual(
+          response.text,
+          (
+              'hello with temperature=0.0, top_k=0.1, top_p=0.2, '
+              "max_tokens=4096, stop=['\\n']."
+          ),
+      )
+      self.assertIsNotNone(response.usage)
+      self.assertIsNotNone(response.usage.prompt_tokens, 2)
+      self.assertIsNotNone(response.usage.completion_tokens, 1)
+      self.assertIsNotNone(response.usage.total_tokens, 3)
+
+  def test_mm_call(self):
+    with mock.patch('requests.Session.post') as mock_mm_request:
+      mock_mm_request.side_effect = mock_mm_requests_post
+      lm = anthropic.Claude3Haiku(api_key='fake_key')
+      response = lm(lf_modalities.Image.from_bytes(image_content), lm=lm)
+      self.assertEqual(response.text, 'image: image/png')
+
+  def test_pdf_call(self):
+    with mock.patch('requests.Session.post') as mock_mm_request:
+      mock_mm_request.side_effect = mock_mm_requests_post
+      lm = anthropic.Claude3Haiku(api_key='fake_key')
+      response = lm(lf_modalities.PDF.from_bytes(pdf_content), lm=lm)
+      self.assertEqual(response.text, 'document: application/pdf')
+
+  def test_call_errors(self):
+    for status_code, error_type, error_message in [
+        (429, 'rate_limit', 'Rate limit exceeded.'),
+        (529, 'service_unavailable', 'Service unavailable.'),
+        (500, 'bad_request', 'Bad request.'),
+    ]:
+      with mock.patch('requests.Session.post') as mock_mm_request:
+        mock_mm_request.side_effect = mock_requests_post_error(
+            status_code, error_type, error_message
+        )
+        lm = anthropic.Claude3Haiku(api_key='fake_key')
+        with self.assertRaisesRegex(
+            Exception, f'.*{status_code}: .*{error_message}'
+        ):
+          lm('hello', max_attempts=1)
+
+
+class VertexAIAnthropicTest(unittest.TestCase):
+  """Tests for VertexAI Anthropic models."""
+
+  def test_basics(self):
+    with self.assertRaisesRegex(ValueError, 'Please specify `project`'):
+      lm = anthropic.VertexAIClaude3_5_Sonnet_20241022()
+      lm('hi')
+
+    model = anthropic.VertexAIClaude3_5_Sonnet_20241022(project='langfun')
+
+    # NOTE(daiyip): For OSS users, default credentials are not available unless
+    # users have already set up their GCP project. Therefore we ignore the
+    # exception here.
+    try:
+      model._initialize()
+    except exceptions.DefaultCredentialsError:
+      pass
+
+    self.assertEqual(
+        model.api_endpoint,
+        (
+            'https://us-east5-aiplatform.googleapis.com/v1/projects/'
+            'langfun/locations/us-east5/publishers/anthropic/'
+            'models/claude-3-5-sonnet-v2@20241022:streamRawPredict'
+        )
+    )
+    request = model.request(
+        lf_message.UserMessage('hi'),
+        language_model.LMSamplingOptions(temperature=0.0),
+    )
+    self.assertEqual(
+        request,
+        {
+            'anthropic_version': 'vertex-2023-10-16',
+            'max_tokens': 8192,
+            'messages': [
+                {'content': [{'text': 'hi', 'type': 'text'}], 'role': 'user'}
+            ],
+            'stream': False,
+            'temperature': 0.0,
+            'top_k': 40,
+        },
+    )
+
+
+if __name__ == '__main__':
+  unittest.main()

langfun/core/llms/cache/base.py

@@ -60,13 +60,16 @@ class LMCacheBase(lf.LMCache):
       self, lm: lf.LanguageModel, prompt: lf.Message, seed: int
   ) -> lf.LMSamplingResult | None:
     """Gets the cached result of a prompt generated by a language model."""
-    entry = self._get(lm.model_id, self._key(lm, prompt, seed))
+    key = self._key(lm, prompt, seed)
+    entry = self._get(lm.model_id, key)
     self._stats.num_queries += 1
     if entry is None:
       self._stats.num_misses += 1
       return None
     if entry.expire is not None and entry.expire < datetime.datetime.now():
       self._stats.num_hit_expires += 1
+      self._stats.num_deletes += 1
+      assert self._delete(lm.model_id, key)
       return None
     self._stats.num_hits += 1
     return entry.result
@@ -86,6 +89,18 @@ class LMCacheBase(lf.LMCache):
     self._put(lm.model_id, self._key(lm, prompt, seed), entry)
     self._stats.num_updates += 1
 
+  def delete(
+      self,
+      lm: lf.LanguageModel,
+      prompt: lf.Message,
+      seed: int,
+  ) -> bool:
+    """Deletes the result of a prompt generated by a language model in cache."""
+    deleted = self._delete(lm.model_id, self._key(lm, prompt, seed))
+    if deleted:
+      self._stats.num_deletes += 1
+    return deleted
+
   @abc.abstractmethod
   def _get(self, model_id: str, key: str) -> LMCacheEntry | None:
     """Returns a LM cache entry associated with the key."""
@@ -94,6 +109,10 @@ class LMCacheBase(lf.LMCache):
   def _put(self, model_id: str, key: str, entry: LMCacheEntry) -> None:
     """Puts a LM cache entry associated with the key."""
 
+  @abc.abstractmethod
+  def _delete(self, model_id: str, key: str) -> bool:
+    """Deletes a LM cache entry associated with the key."""
+
   def _sym_clone(self, deep: bool, memo: Any = None) -> 'LMCacheBase':
     v = super()._sym_clone(deep, memo)
     v._stats = self._stats  # pylint: disable=protected-access
@@ -102,4 +121,4 @@ class LMCacheBase(lf.LMCache):
 
 def default_key(lm: lf.LanguageModel, prompt: lf.Message, seed: int) -> Any:
   """Default key for LM cache."""
-  return (prompt.text, lm.sampling_options.cache_key(), seed)
+  return (prompt.text_with_modality_hash, lm.sampling_options.cache_key(), seed)

langfun/core/llms/cache/in_memory.py

@@ -15,6 +15,7 @@
 
 import collections
 import contextlib
+import json
 from typing import Annotated, Any, Iterator
 import langfun.core as lf
 from langfun.core.llms.cache import base
@@ -49,6 +50,11 @@ class InMemory(base.LMCacheBase):
             "Creating a new cache as cache file '%s' does not exist.",
             self.filename,
         )
+      except json.JSONDecodeError:
+        pg.logging.warning(
+            "Creating a new cache as cache file '%s' is corrupted.",
+            self.filename,
+        )
 
   def model_ids(self) -> list[str]:
     """Returns the model ids of cached queires."""
@@ -99,6 +105,13 @@ class InMemory(base.LMCacheBase):
     """Puts a LM cache entry associated with the key."""
     self._cache[model_id][key] = entry
 
+  def _delete(self, model_id: str, key: str) -> bool:
+    """Deletes a LM cache entry associated with the key."""
+    model_cache = self._cache.get(model_id, None)
+    if model_cache is None:
+      return False
+    return model_cache.pop(key, None) is not None
+
   def reset(self, model_id: str | None = None) -> None:
     """Resets the cache."""
     if model_id is not None:

langfun/core/llms/cache/in_memory_test.py

@@ -44,28 +44,38 @@ class InMemoryLMCacheTest(unittest.TestCase):
     self.assertEqual(
         list(cache.keys()),
         [
-            ('a', (0.0, 1024, 1, 40, None, None), 0),
-            ('a', (0.0, 1024, 1, 40, None, None), 1),
-            ('b', (0.0, 1024, 1, 40, None, None), 0),
-            ('c', (0.0, 1024, 1, 40, None, None), 0),
+            ('a', (None, None, 1, 40, None, None), 0),
+            ('a', (None, None, 1, 40, None, None), 1),
+            ('b', (None, None, 1, 40, None, None), 0),
+            ('c', (None, None, 1, 40, None, None), 0),
         ],
     )
     self.assertEqual(
         list(cache.keys('StaticSequence')),
         [
-            ('a', (0.0, 1024, 1, 40, None, None), 0),
-            ('a', (0.0, 1024, 1, 40, None, None), 1),
-            ('b', (0.0, 1024, 1, 40, None, None), 0),
-            ('c', (0.0, 1024, 1, 40, None, None), 0),
+            ('a', (None, None, 1, 40, None, None), 0),
+            ('a', (None, None, 1, 40, None, None), 1),
+            ('b', (None, None, 1, 40, None, None), 0),
+            ('c', (None, None, 1, 40, None, None), 0),
         ],
     )
 
     def cache_entry(response_text, cache_seed=0):
       return base.LMCacheEntry(
-          lf.LMSamplingResult([
-              lf.LMSample(
-                  lf.AIMessage(response_text, cache_seed=cache_seed), score=1.0)
-          ])
+          lf.LMSamplingResult(
+              [
+                  lf.LMSample(
+                      lf.AIMessage(response_text, cache_seed=cache_seed),
+                      score=1.0,
+                  )
+              ],
+              usage=lf.LMSamplingUsage(
+                  1,
+                  len(response_text),
+                  len(response_text) + 1,
+              ),
+              is_cached=True,
+          )
       )
 
     self.assertEqual(
@@ -90,19 +100,19 @@ class InMemoryLMCacheTest(unittest.TestCase):
         list(cache.items()),
         [
             (
-                ('a', (0.0, 1024, 1, 40, None, None), 0),
+                ('a', (None, None, 1, 40, None, None), 0),
                 cache_entry('1'),
             ),
             (
-                ('a', (0.0, 1024, 1, 40, None, None), 1),
+                ('a', (None, None, 1, 40, None, None), 1),
                 cache_entry('2', 1),
             ),
             (
-                ('b', (0.0, 1024, 1, 40, None, None), 0),
+                ('b', (None, None, 1, 40, None, None), 0),
                 cache_entry('3'),
             ),
             (
-                ('c', (0.0, 1024, 1, 40, None, None), 0),
+                ('c', (None, None, 1, 40, None, None), 0),
                 cache_entry('4'),
             ),
         ],
@@ -111,19 +121,19 @@ class InMemoryLMCacheTest(unittest.TestCase):
         list(cache.items('StaticSequence')),
         [
             (
-                ('a', (0.0, 1024, 1, 40, None, None), 0),
+                ('a', (None, None, 1, 40, None, None), 0),
                 cache_entry('1'),
             ),
             (
-                ('a', (0.0, 1024, 1, 40, None, None), 1),
+                ('a', (None, None, 1, 40, None, None), 1),
                 cache_entry('2', 1),
             ),
             (
-                ('b', (0.0, 1024, 1, 40, None, None), 0),
+                ('b', (None, None, 1, 40, None, None), 0),
                 cache_entry('3'),
             ),
             (
-                ('c', (0.0, 1024, 1, 40, None, None), 0),
+                ('c', (None, None, 1, 40, None, None), 0),
                 cache_entry('4'),
             ),
         ],
@@ -139,6 +149,50 @@ class InMemoryLMCacheTest(unittest.TestCase):
     self.assertIs(copy.deepcopy(cache)._cache, cache._cache)
     self.assertIs(copy.deepcopy(cache)._stats, cache._stats)
 
+    self.assertFalse(
+        cache.delete(fake.StaticResponse('hi'), lf.UserMessage('c'), seed=0)
+    )
+    self.assertFalse(cache.delete(lm, lf.UserMessage('c'), seed=1))
+    self.assertFalse(cache.delete(lm, lf.UserMessage('d'), seed=0))
+    self.assertTrue(cache.delete(lm, lf.UserMessage('c'), seed=0))
+    self.assertEqual(
+        list(cache.keys('StaticSequence')),
+        [
+            ('a', (None, None, 1, 40, None, None), 0),
+            ('a', (None, None, 1, 40, None, None), 1),
+            ('b', (None, None, 1, 40, None, None), 0),
+        ],
+    )
+    self.assertEqual(cache.stats.num_deletes, 1)
+
+  def test_cache_with_modalities(self):
+
+    class CustomModality(lf.Modality):
+      content: str
+
+      def to_bytes(self):
+        return self.content.encode()
+
+    cache = in_memory.InMemory()
+    lm = fake.StaticSequence(['1', '2', '3', '4', '5', '6'], cache=cache)
+    lm(lf.UserMessage('hi <<[[image]]>>', image=CustomModality('foo')))
+    lm(lf.UserMessage('hi <<[[image]]>>', image=CustomModality('bar')))
+    self.assertEqual(
+        list(cache.keys()),
+        [
+            (
+                'hi <<[[image]]>><image>acbd18db</image>',
+                (None, None, 1, 40, None, None),
+                0,
+            ),
+            (
+                'hi <<[[image]]>><image>37b51d19</image>',
+                (None, None, 1, 40, None, None),
+                0,
+            ),
+        ],
+    )
+
   def test_ttl(self):
     cache = in_memory.InMemory(ttl=1)
     lm = fake.StaticSequence(['1', '2', '3'], cache=cache)
@@ -151,6 +205,7 @@ class InMemoryLMCacheTest(unittest.TestCase):
     self.assertEqual(cache.stats.num_hits, 1)
     self.assertEqual(cache.stats.num_hit_expires, 1)
     self.assertEqual(cache.stats.num_misses, 1)
+    self.assertEqual(cache.stats.num_deletes, 1)
 
   def test_different_sampling_options(self):
     cache = in_memory.InMemory()
@@ -161,15 +216,15 @@ class InMemoryLMCacheTest(unittest.TestCase):
     self.assertEqual(
         list(cache.keys()),
         [
-            ('a', (0.0, 1024, 1, 40, None, None), 0),
-            ('a', (1.0, 1024, 1, 40, None, None), 0),
+            ('a', (None, None, 1, 40, None, None), 0),
+            ('a', (1.0, None, 1, 40, None, None), 0),
         ],
     )
 
   def test_different_model(self):
     cache = in_memory.InMemory()
-    lm1 = fake.StaticSequence(['1', '2', '3'], cache=cache)
-    lm2 = fake.Echo(cache=cache)
+    lm1 = fake.StaticSequence(['1', '2', '3'], cache=cache, temperature=0.0)
+    lm2 = fake.Echo(cache=cache, temperature=0.0)
 
     self.assertEqual(lm1('a'), '1')
     self.assertEqual(lm2('a'), 'a')
@@ -180,15 +235,15 @@ class InMemoryLMCacheTest(unittest.TestCase):
     self.assertEqual(
         list(cache.keys('StaticSequence')),
         [
-            ('a', (0.0, 1024, 1, 40, None, None), 0),
-            ('b', (0.0, 1024, 1, 40, None, None), 0),
+            ('a', (0.0, None, 1, 40, None, None), 0),
+            ('b', (0.0, None, 1, 40, None, None), 0),
         ],
     )
     self.assertEqual(
         list(cache.keys('Echo')),
         [
-            ('a', (0.0, 1024, 1, 40, None, None), 0),
-            ('b', (0.0, 1024, 1, 40, None, None), 0),
+            ('a', (0.0, None, 1, 40, None, None), 0),
+            ('b', (0.0, None, 1, 40, None, None), 0),
         ],
     )
     self.assertEqual(len(cache), 4)
@@ -240,6 +295,11 @@ class InMemoryLMCacheTest(unittest.TestCase):
     self.assertEqual(cache2.stats.num_updates, 2)
     cache2.save()
 
+    # Corrupted file.
+    pg.io.writefile(path, 'bad_content')
+    cache3 = in_memory.InMemory(path)
+    self.assertEqual(len(cache3), 0)
+
 
 class LmCacheTest(unittest.TestCase):
 

langfun/core/llms/compositional.py

@@ -0,0 +1,101 @@
+# Copyright 2024 The Langfun Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Compositions of different LLM models."""
+import random
+from typing import Annotated
+
+import langfun.core as lf
+import pyglove as pg
+
+
+@pg.use_init_args(['candidates', 'seed'])
+class RandomChoice(lf.LanguageModel):
+  """Random choice of a list of LLM models."""
+
+  candidates: Annotated[
+      list[lf.LanguageModel],
+      (
+          'A list of LLMs as candidates to choose from.'
+      )
+  ]
+
+  seed: Annotated[
+      int,
+      (
+          'The random seed to use for the random choice.'
+      )
+  ] = 0
+
+  def _on_bound(self):
+    super()._on_bound()
+    self._rand = random.Random(self.seed)
+    # Applying sampling options to all candidates.
+    parent_non_default = self.sampling_options.sym_nondefault()
+    if parent_non_default:
+      for c in self.candidates:
+        c.sampling_options.rebind(
+            parent_non_default, notify_parents=False, raise_on_no_change=False
+        )
+
+  @property
+  def model_id(self) -> str:
+    model_ids = ', '.join(
+        sorted(c.model_id for c in self.candidates)
+    )
+    return f'RandomChoice({model_ids})'
+
+  @property
+  def resource_id(self) -> str:
+    resource_ids = ', '.join(
+        sorted(c.resource_id for c in self.candidates)
+    )
+    return f'RandomChoice({resource_ids})'
+
+  def _select_lm(self) -> lf.LanguageModel:
+    """Selects a random LLM from the candidates."""
+    return self._rand.choice(self.candidates)
+
+  def sample(
+      self,
+      prompts: list[str | lf.Message],
+      *,
+      cache_seed: int = 0,
+      **kwargs,
+  ) -> list[lf.LMSamplingResult]:
+    return self._select_lm().sample(
+        prompts, cache_seed=cache_seed, **kwargs
+    )
+
+  def __call__(
+      self, prompt: lf.Message, *, cache_seed: int = 0, **kwargs
+  ) -> lf.Message:
+    return self._select_lm()(prompt, cache_seed=cache_seed, **kwargs)
+
+  def score(
+      self,
+      prompt: str | lf.Message | list[lf.Message],
+      completions: list[str | lf.Message],
+      **kwargs,
+  ) -> list[lf.LMScoringResult]:
+    return self._select_lm().score(prompt, completions, **kwargs)
+
+  def tokenize(
+      self,
+      prompt: str | lf.Message,
+      **kwargs,
+  ) -> list[tuple[str | bytes, int]]:
+    return self._select_lm().tokenize(prompt, **kwargs)
+
+  def _sample(self, *arg, **kwargs):
+    assert False, 'Should never trigger.'