lalamo 0.3.3__py3-none-any.whl → 0.3.4__py3-none-any.whl

lalamo/__init__.py

@@ -1,7 +1,7 @@
 from lalamo.model_import import REPO_TO_MODEL, ModelSpec, import_model
 from lalamo.modules import Decoder
 
-__version__ = "0.3.3"
+__version__ = "0.3.4"
 
 __all__ = [
     "REPO_TO_MODEL",

lalamo/model_import/common.py

@@ -138,7 +138,13 @@ def import_message_processor(
             raise ValueError("Conflicting chat template specifications.")
         prompt_template = tokenizer_config.chat_template
     tokenizer = Tokenizer.from_file(str(tokenizer_file))
-    tokenizer.add_special_tokens(tokenizer_config.added_tokens())
+
+    added_tokens = tokenizer_config.added_tokens()
+    added_special_tokens = [token for token in added_tokens if token.special]
+    added_not_special_tokens = [token for token in added_tokens if not token.special]
+    tokenizer.add_special_tokens(added_special_tokens)
+    tokenizer.add_tokens(added_not_special_tokens)
+
     message_processor_config = MessageProcessorConfig(
         prompt_template=prompt_template,
         output_parser_regex=model_spec.output_parser_regex,

lalamo/model_import/huggingface_tokenizer_config.py

@@ -73,8 +73,7 @@ class HFTokenizerConfig:
         if self.added_tokens_decoder is None:
             return []
         return [
-            AddedToken(content=token.content, single_word=token.single_word, normalized=token.normalized)
-            for token in self.added_tokens_decoder.values()
+            token.to_added_token() for token in self.added_tokens_decoder.values()
         ]
 
     @classmethod

{lalamo-0.3.3.dist-info → lalamo-0.3.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lalamo
-Version: 0.3.3
+Version: 0.3.4
 Summary: JAX library for optimization and export of models for use with the UZU inference engine.
 Requires-Python: <4,>=3.12
 Description-Content-Type: text/markdown

{lalamo-0.3.3.dist-info → lalamo-0.3.4.dist-info}/RECORD

@@ -1,4 +1,4 @@
-lalamo/__init__.py,sha256=GCS_DTVF-_KdksObU3ceoJbjaxvS4w7fF62TI1NyBh0,217
+lalamo/__init__.py,sha256=cWcHQPJNpb1aArj0uMzq45TRPIzyKfPxC19kiv3mVzs,217
 lalamo/common.py,sha256=5NUFD26yQgOnEEk3LaQnce8n-VwJxILkEpFesHZhtQU,3820
 lalamo/language_model.py,sha256=wZGucNCPP9ReHn6fYO5TlFHTXun1QIMwaxniq9z7K5Q,10639
 lalamo/main.py,sha256=Ta3ZW-Xw3uNPEVvb2YlBOH9UhhLDV7PZjRCg6giu0ao,12275
@@ -8,9 +8,9 @@ lalamo/registry_abc.py,sha256=ENjXiD_wEH100fNjG-W5Em1L_EQ0Lf0pdRhRGvf3qZk,2197
 lalamo/sampling.py,sha256=g_dNiJyZrRqoQIiLid4cr6nRT9N5tSz3GtHr8Bt4n-E,3404
 lalamo/utils.py,sha256=Jm54CKFc6lJSggXVwF_lgiUuKEtQ55c58U-ALU6oxuk,2922
 lalamo/model_import/__init__.py,sha256=Z8pS9rbKKx1QgUy7KZtHxiNWlZhII3mdovT9d37vAxg,168
-lalamo/model_import/common.py,sha256=vVCKRl9-zifMiut4lMjB5eVXOHQxw8yfD5Q7rDr153c,7649
+lalamo/model_import/common.py,sha256=puMyMm2HivQ_qUt6iPcAcdzq6wal93owyBg88B3EVks,7906
 lalamo/model_import/huggingface_generation_config.py,sha256=mot6VQ6ezCtEhN6VjhnvaU-nR5P5T2BuBUgpFNnWJxU,1495
-lalamo/model_import/huggingface_tokenizer_config.py,sha256=kTdgH2lle1m7zRtClO3BhgonvfZsAjeDGjPMAd1EU8E,2607
+lalamo/model_import/huggingface_tokenizer_config.py,sha256=34oZiY9RQVAYSPFSM0-epwr2Su3XzipSUl4V9dz71PU,2524
 lalamo/model_import/decoder_configs/__init__.py,sha256=Ru_lI1IjahyXZBSB5J0eWwVM6gn2ilRZW65m2sU2hMw,460
 lalamo/model_import/decoder_configs/common.py,sha256=L1oUilS4KwfnwKyWegoSZ3Tha5CqRjN6gjMhCie4Ymk,1849
 lalamo/model_import/decoder_configs/executorch.py,sha256=Kx_T-B5jumfWf9vj20We4FF0GkSkTmIYeWOss88-qYA,5266
@@ -51,9 +51,9 @@ lalamo/modules/normalization.py,sha256=YecmA0hnVVYd473FzVgxunHYA377NWMKZRfuQNB8D
 lalamo/modules/rope.py,sha256=IlL4Jz4fw6eTKXXMls1BXBAgbuB3GnP0aVB9GrF1nA8,10012
 lalamo/modules/torch_interop.py,sha256=-mujd1zI4ec2w92Hd50RtDa0K3jl6ZSnPxc5r3Fp9nU,916
 lalamo/modules/utils.py,sha256=5QTdi34kEI5jix7TfTdB0mOYZbzZUul_T1y8eWCA6lQ,262
-lalamo-0.3.3.dist-info/licenses/LICENSE,sha256=diHRfjSEJHD1nnEeMIfMRCjR3UERf8bT3eseD6b1ayA,1072
-lalamo-0.3.3.dist-info/METADATA,sha256=sngvnwPijSDHmBBtGzZ4-KHVg0UB99oE7fK6aPVPO3I,2854
-lalamo-0.3.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lalamo-0.3.3.dist-info/entry_points.txt,sha256=qli7qTfnBk5WP10rOGXXEckHMtt-atJMDWd8jN89Uks,43
-lalamo-0.3.3.dist-info/top_level.txt,sha256=VHvWL5JN5XRG36NsN_MieJ7EwRihEOrEjyDaTdFJ-aI,7
-lalamo-0.3.3.dist-info/RECORD,,
+lalamo-0.3.4.dist-info/licenses/LICENSE,sha256=diHRfjSEJHD1nnEeMIfMRCjR3UERf8bT3eseD6b1ayA,1072
+lalamo-0.3.4.dist-info/METADATA,sha256=YnFSMWXzI5YyOSHGa8wWJ9QUaO5rJhnGS7GIUo9QDBM,2854
+lalamo-0.3.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lalamo-0.3.4.dist-info/entry_points.txt,sha256=qli7qTfnBk5WP10rOGXXEckHMtt-atJMDWd8jN89Uks,43
+lalamo-0.3.4.dist-info/top_level.txt,sha256=VHvWL5JN5XRG36NsN_MieJ7EwRihEOrEjyDaTdFJ-aI,7
+lalamo-0.3.4.dist-info/RECORD,,