onnx-diagnostic 0.8.10__py3-none-any.whl → 0.9.0__py3-none-any.whl

onnx_diagnostic/export/api.py

@@ -428,6 +428,16 @@ class WrapperToExportMethodToOnnx(torch.nn.Module):
                 new_kwargs[k] = v
         return new_kwargs
 
+    def is_empty_cache(self, cache):
+        if cache.__class__.__name__ == "DynamicCache" and hasattr(cache, "layers"):
+            if len(cache.layers) == 1 and cache.layers[0].keys is None:
+                return True
+            if len(cache.layers) == 0:
+                return True
+        if cache is None:
+            return True
+        return False
+
     def forward(self, *args, **kwargs):
         if not self._export_done:
             inp_args = args
@@ -443,6 +453,7 @@ class WrapperToExportMethodToOnnx(torch.nn.Module):
                     if v is not None
                     and (not self.skip_kwargs_names or k not in self.skip_kwargs_names)
                     and not isinstance(v, (bool, int, float))
+                    and not self.is_empty_cache(v)
                 }
             )
             inp_args, inp_kwargs = torch_deepcopy((inp_args, inp_kwargs))
@@ -509,12 +520,10 @@ class WrapperToExportMethodToOnnx(torch.nn.Module):
             simple_sig = inspect.Signature(params, return_annotation=inspect._empty)
             args = str(simple_sig)[1:-1]
             calls_args = ", ".join(f"{p}={p}" for p in simple_sig.parameters)
-            src = textwrap.dedent(
-                f"""
+            src = textwrap.dedent(f"""
                 def f(self, {args}):
                     return self._method_call({calls_args})
-                """
-            )
+                """)
             self._method_src = src
             ns = {}
             try:

onnx_diagnostic/export/dynamic_shapes.py

@@ -834,7 +834,7 @@ class ModelInputs:
         """Guesses the dynamic shapes for one argument."""
         if len(objs) == 0:
             return None
-        set_types = set(type(o) for o in objs)
+        set_types = set(type(o) for o in objs if o is not None)
         assert (
             len(set_types) == 1
         ), f"Unexpected variety of input type {set_types}{msg() if msg else ''})"

onnx_diagnostic/export/validate.py

@@ -80,6 +80,7 @@ def compare_modules(
         )
     got = modep(*_get(args), **_get(kwargs))
     if verbose:
+        # pyrefly: ignore[unbound-name]
         d = time.perf_counter() - begin
         print(f"[compare_modules] done in {d} with output={string_type(got, with_shape=True)}")
     if mod:
@@ -89,6 +90,7 @@ def compare_modules(
         expected = mod(*_get(args), **_get(kwargs))
         diff = max_diff(expected, got)
         if verbose:
+            # pyrefly: ignore[unbound-name]
             d = time.perf_counter() - begin
             print(
                 f"[compare_modules] done in {d} with "

onnx_diagnostic/ext_test_case.py

@@ -780,7 +780,7 @@ class ExtTestCase(unittest.TestCase):
 
     @property
     def verbose(self) -> int:
-        "Returns the the value of environment variable ``VERBOSE``."
+        "Returns the value of environment variable ``VERBOSE``."
         return int(os.environ.get("VERBOSE", "0"))
 
     @classmethod
@@ -1028,6 +1028,19 @@ class ExtTestCase(unittest.TestCase):
                 rtol=rtol,
                 msg=msg,
             )
+        elif expected.__class__.__name__ == "BaseModelOutputWithPooling":
+            if expected.__class__.__name__ == value.__class__.__name__:
+                self.assertEqual(len(expected), len(value), msg=msg)
+                self.assertEqual(list(expected), list(value), msg=msg)  # checks the order
+                self.assertEqualAny(
+                    {k: v for k, v in expected.items()},  # noqa: C416
+                    {k: v for k, v in value.items()},  # noqa: C416
+                    atol=atol,
+                    rtol=rtol,
+                    msg=msg,
+                )
+            else:
+                self.assertEqualArray(expected.last_hidden_state, value)
         elif isinstance(expected, (tuple, list, dict)):
             self.assertIsInstance(value, type(expected), msg=msg)
             self.assertEqual(len(expected), len(value), msg=msg)
@@ -1043,24 +1056,28 @@ class ExtTestCase(unittest.TestCase):
             "SlidingWindowCache",
             "HybridCache",
         ):
+            from .helpers.cache_helper import CacheKeyValue
+
             self.assertEqual(type(expected), type(value), msg=msg)
-            atts = ["key_cache", "value_cache"]
-            self.assertEqualAny(
-                {k: expected.__dict__.get(k, None) for k in atts},
-                {k: value.__dict__.get(k, None) for k in atts},
-                atol=atol,
-                rtol=rtol,
-            )
+            self.assertEqualAny(CacheKeyValue(expected), CacheKeyValue(value))
         elif expected.__class__.__name__ == "StaticCache":
+            from .helpers.cache_helper import CacheKeyValue
+
             self.assertEqual(type(expected), type(value), msg=msg)
             self.assertEqual(expected.max_cache_len, value.max_cache_len)
-            atts = ["key_cache", "value_cache"]
-            self.assertEqualAny(
-                {k: expected.__dict__.get(k, None) for k in atts},
-                {k: value.__dict__.get(k, None) for k in atts},
-                atol=atol,
-                rtol=rtol,
-            )
+            self.assertEqualAny(CacheKeyValue(expected), CacheKeyValue(value))
+        elif expected.__class__.__name__ == "CacheKeyValue":
+            self.assertEqual(type(expected), type(value), msg=msg)
+            if expected.cls_layers is None:
+                self.assertEqual(expected.cls_layers, value.cls_layers)
+            else:
+                self.assertEqualAny(
+                    [cls.__name__ for cls in expected.cls_layers],
+                    [cls.__name__ for cls in value.cls_layers],
+                    msg=msg,
+                )
+            self.assertEqualAny(expected.key_cache, value.key_cache, msg=msg)
+            self.assertEqualAny(expected.value_cache, value.value_cache, msg=msg)
         elif expected.__class__.__name__ == "EncoderDecoderCache":
             self.assertEqual(type(expected), type(value), msg=msg)
             atts = ["self_attention_cache", "cross_attention_cache"]

onnx_diagnostic/helpers/args_helper.py

@@ -105,6 +105,7 @@ def get_parsed_args(
                 default=tries,
             )
     for k, v in kwargs.items():
+        assert isinstance(v, tuple)  # type
         parser.add_argument(
             f"--{k}",
             help=f"{v[1]}, default is {v[0]}",

onnx_diagnostic/helpers/bench_run.py

@@ -11,7 +11,6 @@ from argparse import Namespace
 from datetime import datetime
 from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union
 
-
 _DEFAULT_STRING_LIMIT = 2000
 
 

onnx_diagnostic/helpers/cache_helper.py

@@ -4,6 +4,19 @@ import torch
 import transformers
 import transformers.cache_utils
 
+KWARGS_LAYER = {}
+if hasattr(transformers.cache_utils, "DynamicSlidingWindowLayer"):
+    KWARGS_LAYER.update(
+        {
+            transformers.cache_utils.DynamicSlidingWindowLayer: lambda tensor: {
+                "sliding_window": tensor.shape[2]
+            },
+            transformers.cache_utils.StaticSlidingWindowLayer: lambda tensor: {
+                "sliding_window": tensor.shape[2]
+            },
+        }
+    )
+
 
 class CacheKeyValue:
     """
@@ -90,7 +103,7 @@ def flatten_unflatten_for_dynamic_shapes(
         the context gives the dictionary keys but it is not expressed
         in the dynamic shapes, these specifications seems to be different
         for the strict and non strict mode. It also preserves tuple.
-    :param change_function: to modifies the tensor in the structure itself,
+    :param change_function: to modify the tensor in the structure itself,
         like replace them by a shape
     :return: the serialized object
     """
@@ -110,7 +123,7 @@ def flatten_unflatten_for_dynamic_shapes(
         start = end
     if use_dict:
         if spec.type is dict:
-            # This a dictionary.
+            # This is a dictionary.
             return dict(zip(spec.context, subtrees))
         if spec.type is tuple:
             return tuple(subtrees)
@@ -185,6 +198,7 @@ if pv.Version(transformers.__version__) > pv.Version("4.49.99999"):
     def make_dynamic_cache(
         key_value_pairs: Union[List[torch.Tensor], List[Tuple[torch.Tensor, torch.Tensor]]],
         cls_layers: Optional[Union[str, List[type]]] = None,
+        cls_kwargs: Optional[Union[Dict[str, int], List[Dict[str, int]]]] = None,
     ) -> transformers.cache_utils.DynamicCache:
         """
         Creates an instance of :class:`transformers.cache_utils.DynamicCache`.
@@ -194,6 +208,8 @@ if pv.Version(transformers.__version__) > pv.Version("4.49.99999"):
         :param cls_layers: to select the appropriate class to use on each layer,
             if specified, sliding_window is ignored, it can be a string
             if all layers are expected to follow the same class
+        :param cls_kwargs: arguments used to build a specific layer,
+            such as ``sliding_window`` for ``DynamicSlidingWindowLayer``
         :return: :class:`transformers.cache_utils.DynamicCache`
 
         Example:
@@ -224,49 +240,70 @@ if pv.Version(transformers.__version__) > pv.Version("4.49.99999"):
         are supported.
         """
         key_value_pairs = _preprocess_key_value_pairs(key_value_pairs)
-        cls_kwargs = {}
         if isinstance(cls_layers, str):
             assert hasattr(
                 transformers.cache_utils, cls_layers
-            ), f"Unable to find class {cls_layers!r} in transformers.cache_utils"
-            cls_layer = getattr(transformers.cache_utils, cls_layers)
-            if cls_layers == "DynamicSlidingWindowLayer":
-                cls_kwargs["sliding_window"] = key_value_pairs[0][0].shape[2]
-                assert isinstance(
-                    cls_kwargs["sliding_window"], int
-                ), f"sliding_window must be an integer but shape={key_value_pairs[0][0].shape}"
-        elif cls_layers is not None:
-            unique = set(cls_layers)
-            assert len(unique) == 1, f"Not implemented when cls_layers={cls_layers}"
-            cls_layer = unique.pop()
-            if (
-                hasattr(transformers.cache_utils, "DynamicSlidingWindowLayer")
-                and cls_layer == transformers.cache_utils.DynamicSlidingWindowLayer
-            ):
-                from .helper import string_type
-
-                assert key_value_pairs and key_value_pairs[0], (
-                    f"not implemented for key_value_pairs="
-                    f"{string_type(key_value_pairs, with_shape=True)}"
-                )
-                cls_kwargs["sliding_window"] = key_value_pairs[0][0].shape[2]
-                assert isinstance(
-                    cls_kwargs["sliding_window"], int
-                ), f"sliding_window must be an integer but shape={key_value_pairs[0][0].shape}"
+            ), f"Missing layer class {cls_layers!r}"
+            cls_layers = getattr(transformers.cache_utils, cls_layers)
+        if cls_layers and not isinstance(cls_layers, list):
+            cls_layers = [cls_layers for _ in key_value_pairs]  # type: ignore[misc]
+        if cls_layers is not None and isinstance(cls_layers, list):
+            assert len(cls_layers) == len(key_value_pairs), (
+                f"Length mismatch {len(key_value_pairs)} expected but "
+                f"{len(cls_layers)} layer types are given."
+            )
+            if cls_kwargs is None:
+                cls_kwargs = [{} for _kv in key_value_pairs]  # type: ignore[assignment]
+            assert len(cls_layers) == len(cls_kwargs), (
+                f"Length mismatch {len(cls_kwargs)} expected but "
+                f"{len(cls_layers)} layer types are given, "
+                f"cls_layers={cls_layers}, cls_kwargs={cls_kwargs}"
+            )
+            cls_layer = None
+            assert (
+                key_value_pairs and key_value_pairs[0]
+            ), f"not implemented for type(key_value_pairs[0])={type(key_value_pairs[0])}"
+            for kv, clsy, kws in zip(key_value_pairs, cls_layers, cls_kwargs):
+                default_values = KWARGS_LAYER.get(clsy, lambda tensor: {})(kv[0])
+                for k, v in default_values.items():
+                    if k not in kws:
+                        kws[k] = v  # type: ignore[index]
         else:
+            assert cls_kwargs is None, "cls_layers must be a list if cls_kwargs is specified"
+            assert (
+                cls_layers is None
+            ), f"cls_layers must be list or a string but it is {cls_layers}"
+            cls_kwargs = {}
             cls_layer = (
                 transformers.cache_utils.DynamicLayer
                 if hasattr(transformers.cache_utils, "DynamicLayer")
                 else None
             )
 
+        if cls_layer is not None:
+            assert isinstance(cls_kwargs, dict), (
+                f"one layer = one set of arguments, cls_layer={cls_layer}, "
+                f"cls_kwargs={cls_kwargs}"
+            )
+            cls_layers = [cls_layer for _ in key_value_pairs]
+            cls_kwargs = (
+                cls_kwargs  # type: ignore[assignment]
+                if isinstance(cls_kwargs, list)
+                else [cls_kwargs for _ in key_value_pairs]
+            )
+        elif cls_layers is not None:
+            assert isinstance(cls_layers, list), f"Unexpected type cls_layers={cls_layers}"
+            assert isinstance(cls_kwargs, list), f"Unexpected type cls_kwargs={cls_kwargs}"
+
         if (
             key_value_pairs
             and isinstance(key_value_pairs[0][0], torch._subclasses.fake_tensor.FakeTensor)
             and pv.Version(transformers.__version__) >= pv.Version("4.56")
         ):
             cache = transformers.cache_utils.DynamicCache()
-            cache.layers.extend([cls_layer(**cls_kwargs) for _ in key_value_pairs])
+            cache.layers.extend(
+                [cls_layer(**kws) for cls_layer, kws in zip(cls_layers, cls_kwargs)]  # type: ignore[operator, arg-type]
+            )
             for i, layer in enumerate(cache.layers):
                 k, v = key_value_pairs[i][0], key_value_pairs[i][1]
                 layer.dtype = k.dtype
@@ -281,8 +318,25 @@ if pv.Version(transformers.__version__) > pv.Version("4.49.99999"):
             return finalize_cache(cache)
 
         cache = transformers.cache_utils.DynamicCache()
-        if hasattr(cache, "layers") and cls_layer != transformers.cache_utils.DynamicLayer:
-            cache.layers.extend([cls_layer(**cls_kwargs) for _ in key_value_pairs])
+        if hasattr(cache, "layers") and (
+            cls_layer is None or cls_layer != transformers.cache_utils.DynamicLayer
+        ):
+            assert isinstance(cls_layers, list) and isinstance(cls_kwargs, list), (
+                f"Wrong type {type(cls_layers)} for cls_layers or "
+                f"{type(cls_kwargs)} for cls_kwargs"
+            )
+            assert len(cls_kwargs) == len(cls_layers) and len(cls_kwargs) == len(
+                key_value_pairs
+            ), (
+                f"Length mismatch between len(cls_kwargs)={len(cls_kwargs)}, "
+                f"len(cls_layers)={len(cls_layers)}, "
+                f"len(key_value_pairs)={len(key_value_pairs)}, "
+                f"cls_kwargs={cls_kwargs}, cls_layers={cls_layers}"
+            )
+            del cache.layers[:]
+            cache.layers.extend(
+                [cls_layer(**kws) for cls_layer, kws in zip(cls_layers, cls_kwargs)]  # type: ignore[operator, arg-type]
+            )
             for i, layer in enumerate(cache.layers):
                 layer.keys, layer.values = key_value_pairs[i][0], key_value_pairs[i][1]
                 layer.is_initialized = True
@@ -306,6 +360,7 @@ else:
     def make_dynamic_cache(
         key_value_pairs: Union[List[torch.Tensor], List[Tuple[torch.Tensor, torch.Tensor]]],
         cls_layers: Optional[Union[str, List[type]]] = None,
+        cls_kwargs: Optional[Union[Dict[str, int], List[Dict[str, int]]]] = None,
     ) -> transformers.cache_utils.DynamicCache:
         """
         Creates an instance of :class:`transformers.cache_utils.DynamicCache`.
@@ -337,7 +392,9 @@ else:
             )
             print(string_type(past_key_values, with_shape=True))
         """
-        assert not cls_layers, "cls_layers cannot be used for transformers<5."
+        assert (
+            not cls_layers and not cls_kwargs
+        ), "cls_layers, cls_kwargs cannot be used for transformers<5."
         key_value_pairs = _preprocess_key_value_pairs(key_value_pairs)
         cache = transformers.cache_utils.DynamicCache(len(key_value_pairs))  # type: ignore
         for i, (key, value) in enumerate(key_value_pairs):
@@ -348,6 +405,7 @@ else:
 def make_static_cache(
     key_value_pairs: Union[List[torch.Tensor], List[Tuple[torch.Tensor, torch.Tensor]]],
     max_cache_len: Optional[int] = None,
+    cls_layers: Optional[Union[str, List[type]]] = None,
 ) -> transformers.cache_utils.DynamicCache:
     """
     Creates an instance of :class:`transformers.cache_utils.StaticCache`.
@@ -379,6 +437,9 @@ def make_static_cache(
         )
         print(string_type(past_key_values, with_shape=True))
     """
+    assert not cls_layers or set(cls_layers) == {
+        transformers.cache_utils.StaticLayer
+    }, f"Not implemented when cls_layers={cls_layers!r}"
     key_value_pairs = _preprocess_key_value_pairs(key_value_pairs)
 
     class _config:
@@ -583,13 +644,9 @@ if hasattr(transformers.cache_utils, "SlidingWindowCache"):
         )
         return finalize_cache(cache)
 
-    def get_make_hybrid_cache():
-        return make_sliding_window_cache
-
 else:
     make_sliding_window_cache = None  # type: ignore[assignment]
 
-
 if hasattr(transformers.cache_utils, "HybridCache"):
 
     def make_hybrid_cache(
@@ -775,4 +832,13 @@ def finalize_cache(cache: transformers.cache_utils.Cache) -> transformers.cache_
         # This is used to expand the cache when it does not contains enough layers.
         # This is needed since transformers>4.55.3
         cache.layer_class_to_replicate = cache.layers[0].__class__
+    assert (
+        not hasattr(cache, "layers")
+        or len(cache.layers) != 1
+        or cache.layers[0].keys is not None
+    ), (
+        f"Size mismatch between {len(cache.layers)=}, "
+        f"first key={cache.layers[0].keys}, "  # type: ignore[attr-defined]
+        f"first value={cache.layers[0].values}"  # type: ignore[attr-defined]
+    )
     return cache

onnx_diagnostic/helpers/doc_helper.py

@@ -1,5 +1,5 @@
 import os
-from typing import Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple
 import onnx
 import onnx.helper as oh
 import torch
@@ -46,10 +46,10 @@ class LayerNormalizationOrt(OpRunKernel):
             f"This kernel implementation only work when only one output "
             f"is required but {node.output} were."
         )
-        self._cache: Dict[Tuple[int, int], onnx.ModelProto] = {}
+        self._cache: Dict[Tuple[int, int], Any] = {}
         self.is_cpu = torch.device("cpu") == self.device
 
-    def _make_model(self, itype: int, rank: int, has_bias: bool) -> onnx.ModelProto:
+    def _make_model(self, itype: int, rank: int, has_bias: bool) -> Any:
         shape = [*["d{i}" for i in range(rank - 1)], "last"]
         layer_model = oh.make_model(
             oh.make_graph(
@@ -88,6 +88,7 @@ class LayerNormalizationOrt(OpRunKernel):
             providers=[provider],
         )
 
+    # pyrefly: ignore[bad-override]
     def run(self, x, scale, bias=None):
         itype = torch_dtype_to_onnx_dtype(x.dtype)
         rank = len(x.shape)
@@ -124,7 +125,7 @@ class MatMulOrt(OpRunKernel):
         self._cache: Dict[Tuple[int, int, int], onnx.ModelProto] = {}
         self.is_cpu = torch.device("cpu") == self.device
 
-    def _make_model(self, itype: int, ranka: int, rankb: int) -> onnx.ModelProto:
+    def _make_model(self, itype: int, ranka: int, rankb: int) -> Any:
         shapea = ["a{i}" for i in range(ranka)]
         shapeb = ["b{i}" for i in range(rankb)]
         shapec = ["c{i}" for i in range(max(ranka, rankb))]
@@ -149,6 +150,7 @@ class MatMulOrt(OpRunKernel):
             providers=[provider],
         )
 
+    # pyrefly: ignore[bad-override]
     def run(self, a, b):
         itype = torch_dtype_to_onnx_dtype(a.dtype)
         ranka, rankb = len(a.shape), len(b.shape)
@@ -159,5 +161,6 @@ class MatMulOrt(OpRunKernel):
         if self.verbose:
             print(f"[MatMulOrt] running on {self._provider!r}")
         feeds = dict(A=a.tensor, B=b.tensor)
+        # pyrefly: ignore[missing-attribute]
         got = sess.run(None, feeds)[0]
         return OpRunTensor(got)

onnx_diagnostic/helpers/graph_helper.py

@@ -36,7 +36,7 @@ class GraphRendering:
         :return: computation order
         """
         assert not ({"If", "Scan", "Loop", "SequenceMap"} & set(n.op_type for n in nodes)), (
-            f"This algorithme is not yet implemented if the sequence contains "
+            f"This algorithm is not yet implemented if the sequence contains "
             f"a control flow, types={sorted(set(n.op_type for n in nodes))}"
         )
         number = {e: start - 1 for e in (existing or [])}  # noqa: C420
@@ -131,14 +131,14 @@ class GraphRendering:
     @property
     def nodes(self) -> List[onnx.NodeProto]:
         "Returns the list of nodes"
-        return (
+        return list(
             self.proto.graph.node
             if isinstance(self.proto, onnx.ModelProto)
             else self.proto.node
         )
 
     @property
-    def start_names(self) -> List[onnx.NodeProto]:
+    def start_names(self) -> List[str]:
         "Returns the list of known names, inputs and initializer"
         graph = self.proto.graph if isinstance(self.proto, onnx.ModelProto) else self.proto
         input_names = (
@@ -151,7 +151,7 @@ class GraphRendering:
             if isinstance(graph, onnx.FunctionProto)
             else [
                 *[i.name for i in graph.initializer],
-                *[i.name for i in graph.sparse_initializer],
+                *[i.values.name for i in graph.sparse_initializer],
             ]
         )
         return [*input_names, *init_names]
@@ -159,7 +159,7 @@ class GraphRendering:
     @property
     def input_names(self) -> List[str]:
         "Returns the list of input names."
-        return (
+        return list(
             self.proto.input
             if isinstance(self.proto, onnx.FunctionProto)
             else [
@@ -173,7 +173,7 @@ class GraphRendering:
     @property
     def output_names(self) -> List[str]:
         "Returns the list of output names."
-        return (
+        return list(
             self.proto.output
             if isinstance(self.proto, onnx.FunctionProto)
             else [

onnx_diagnostic/helpers/helper.py

@@ -574,6 +574,32 @@ def string_type(
             print(f"[string_type] CACHE1:{type(obj)}")
         return f"MambaCache(conv_states={c}, ssm_states={d})"
 
+    if (
+        obj.__class__.__name__ in {"DynamicCache"}
+        and hasattr(obj, "layers")
+        and any(lay.__class__.__name__ != "DynamicLayer" for lay in obj.layers)
+    ):
+        slay = []
+        for lay in obj.layers:
+            skeys = string_type(
+                lay.keys,
+                with_shape=with_shape,
+                with_min_max=with_min_max,
+                with_device=with_device,
+                limit=limit,
+                verbose=verbose,
+            )
+            svalues = string_type(
+                lay.keys,
+                with_shape=with_shape,
+                with_min_max=with_min_max,
+                with_device=with_device,
+                limit=limit,
+                verbose=verbose,
+            )
+            slay.append(f"{lay.__class__.__name__}({skeys}, {svalues})")
+        return f"{obj.__class__.__name__}({', '.join(slay)})"
+
     if obj.__class__.__name__ in {
         "DynamicCache",
         "SlidingWindowCache",
@@ -829,6 +855,19 @@ def string_type(
         return f"{obj}"
     if obj.__class__.__name__ == "FakeTensorContext":
         return "FakeTensorContext(...)"
+    if obj.__class__.__name__ == "Chat":
+        import transformers.utils.chat_template_utils as ctu
+
+        assert isinstance(obj, ctu.Chat), f"unexpected type {type(obj)}"
+        msg = string_type(
+            obj.messages,
+            with_shape=with_shape,
+            with_min_max=with_min_max,
+            with_device=with_device,
+            limit=limit,
+            verbose=verbose,
+        )
+        return f"Chat({msg})"
 
     if verbose:
         print(f"[string_type] END:{type(obj)}")