keras-hub-nightly 0.22.0.dev202508190411__py3-none-any.whl → 0.23.0.dev202508200409__py3-none-any.whl

keras_hub/src/models/backbone.py

@@ -277,3 +277,19 @@ class Backbone(keras.Model):
             layer.lora_kernel_a.assign(lora_kernel_a)
             layer.lora_kernel_b.assign(lora_kernel_b)
         store.close()
+
+    def export_to_transformers(self, path):
+        """Export the backbone model to HuggingFace Transformers format.
+
+        This saves the backbone's configuration and weights in a format
+        compatible with HuggingFace Transformers. For unsupported model
+        architectures, a ValueError is raised.
+
+        Args:
+            path: str. Path to save the exported model.
+        """
+        from keras_hub.src.utils.transformers.export.hf_exporter import (
+            export_backbone,
+        )
+
+        export_backbone(self, path)

keras_hub/src/models/causal_lm.py

@@ -392,3 +392,24 @@ class CausalLM(Task):
             outputs = [postprocess(x) for x in outputs]
 
         return self._normalize_generate_outputs(outputs, input_is_scalar)
+
+    def export_to_transformers(self, path):
+        """Export the full CausalLM model to HuggingFace Transformers format.
+
+        This exports the trainable model, tokenizer, and configurations in a
+        format compatible with HuggingFace Transformers. For unsupported model
+        architectures, a ValueError is raised.
+
+        If the preprocessor is attached (default), both the trainable model and
+        tokenizer are exported. To export only the trainable model, set
+        `self.preprocessor = None` before calling this method, then export the
+        preprocessor separately via `preprocessor.export_to_transformers(path)`.
+
+        Args:
+            path: str. Path to save the exported model.
+        """
+        from keras_hub.src.utils.transformers.export.hf_exporter import (
+            export_to_safetensors,
+        )
+
+        export_to_safetensors(self, path)

keras_hub/src/models/causal_lm_preprocessor.py

@@ -180,3 +180,17 @@ class CausalLMPreprocessor(Preprocessor):
         self._sequence_length = value
         if self.packer is not None:
             self.packer.sequence_length = value
+
+    def export_to_transformers(self, path):
+        """Export the preprocessor to HuggingFace Transformers format.
+
+        Args:
+            path: str. Path to save the exported preprocessor/tokenizer.
+        """
+        if self.tokenizer is None:
+            raise ValueError("Preprocessor must have a tokenizer for export.")
+        from keras_hub.src.utils.transformers.export.hf_exporter import (
+            export_tokenizer,
+        )
+
+        export_tokenizer(self.tokenizer, path)

keras_hub/src/tokenizers/tokenizer.py

@@ -261,3 +261,18 @@ class Tokenizer(PreprocessingLayer):
         if cls.backbone_cls != backbone_cls:
             cls = find_subclass(preset, cls, backbone_cls)
         return loader.load_tokenizer(cls, config_file, **kwargs)
+
+    def export_to_transformers(self, path):
+        """Export the tokenizer to HuggingFace Transformers format.
+
+        This saves tokenizer assets in a format compatible with HuggingFace
+        Transformers.
+
+        Args:
+            path: str. Path to save the exported tokenizer.
+        """
+        from keras_hub.src.utils.transformers.export.hf_exporter import (
+            export_tokenizer,
+        )
+
+        export_tokenizer(self, path)

keras_hub/src/utils/transformers/export/gemma.py

@@ -2,6 +2,7 @@ import keras.ops as ops
 
 
 def get_gemma_config(backbone):
+    token_embedding_layer = backbone.get_layer("token_embedding")
     hf_config = {
         "vocab_size": backbone.vocabulary_size,
         "num_hidden_layers": backbone.num_layers,
@@ -11,11 +12,16 @@ def get_gemma_config(backbone):
         "intermediate_size": backbone.intermediate_dim // 2,
         "head_dim": backbone.head_dim,
         "max_position_embeddings": 8192,
+        "tie_word_embeddings": token_embedding_layer.tie_weights,
+        "pad_token_id": 0,
+        "bos_token_id": 2,
+        "eos_token_id": 1,
+        "model_type": "gemma",
     }
     return hf_config
 
 
-def get_gemma_weights_map(backbone):
+def get_gemma_weights_map(backbone, include_lm_head=False):
     weights_dict = {}
 
     # Map token embedding
@@ -83,7 +89,46 @@ def get_gemma_weights_map(backbone):
         "final_normalization"
     ).weights[0]
 
-    # Tie weights, but clone to avoid sharing memory issues
-    weights_dict["lm_head.weight"] = ops.copy(token_embedding_layer.weights[0])
-
+    # Map lm_head if embeddings are not tied
+    if include_lm_head and not token_embedding_layer.tie_weights:
+        weights_dict["lm_head.weight"] = ops.transpose(
+            token_embedding_layer.reverse_embeddings
+        )
     return weights_dict
+
+
+def get_gemma_tokenizer_config(tokenizer):
+    tokenizer_config = {
+        "tokenizer_class": "GemmaTokenizer",
+        "clean_up_tokenization_spaces": False,
+        "bos_token": "<bos>",
+        "eos_token": "<eos>",
+        "pad_token": "<pad>",
+        "unk_token": "<unk>",
+        "add_bos_token": True,
+        "add_eos_token": False,
+        "model_max_length": 8192,
+    }
+    # Add added_tokens_decoder
+    added_tokens_decoder = {}
+    special_tokens = [
+        "<pad>",
+        "<bos>",
+        "<eos>",
+        "<unk>",
+        "<start_of_turn>",
+        "<end_of_turn>",
+    ]
+    for token in special_tokens:
+        token_id = tokenizer.token_to_id(token)
+        if token_id is not None:
+            added_tokens_decoder[str(token_id)] = {
+                "content": token,
+                "special": True,
+                "single_word": False,
+                "lstrip": False,
+                "rstrip": False,
+                "normalized": False,
+            }
+    tokenizer_config["added_tokens_decoder"] = added_tokens_decoder
+    return tokenizer_config

keras_hub/src/utils/transformers/export/hf_exporter.py

@@ -6,58 +6,59 @@ import warnings
 import keras
 
 from keras_hub.src.utils.transformers.export.gemma import get_gemma_config
+from keras_hub.src.utils.transformers.export.gemma import (
+    get_gemma_tokenizer_config,
+)
 from keras_hub.src.utils.transformers.export.gemma import get_gemma_weights_map
 
 MODEL_CONFIGS = {
     "GemmaBackbone": get_gemma_config,
-    # Add future models here, e.g., "LlamaBackbone": get_llama_config,
+    # Add for future models, e.g., "MistralBackbone": get_mistral_config
 }
 
 MODEL_EXPORTERS = {
     "GemmaBackbone": get_gemma_weights_map,
-    # Add future models here, e.g., "LlamaBackbone": get_llama_weights_map,
+    # Add for future models, e.g., "MistralBackbone": get_mistral_weights_map
 }
 
+MODEL_TOKENIZER_CONFIGS = {
+    "GemmaTokenizer": get_gemma_tokenizer_config,
+    # Add for future models, e.g., "MistralTokenizer":
+    # get_mistral_tokenizer_config
+}
 
-def export_to_safetensors(keras_model, path):
-    """Converts a Keras model to Hugging Face safetensor format.
 
-    It does the following:
-    - Extracts and maps weights from the Keras backbone to safetensors.
-    - Saves the configuration as 'config.json'.
-    - Saves weights in 'model.safetensors'.
-    - Saves tokenizer assets.
+def export_backbone(backbone, path, include_lm_head=False):
+    """Export the backbone model to HuggingFace format.
 
     Args:
-        keras_model: The Keras model to convert.
-        path: str. Path of the directory to which the safetensors file,
-          config and tokenizer will be saved.
+        backbone: The Keras backbone model to convert.
+        path: str. Path to save the exported model.
+        include_lm_head: bool. If True, include lm_head weights if applicable.
     """
     backend = keras.config.backend()
-    backbone = keras_model.backbone
     model_type = backbone.__class__.__name__
-
     if model_type not in MODEL_CONFIGS:
-        raise ValueError(f"Config not implemented for {model_type}")
-
+        raise ValueError(
+            f"Export to Transformers format not implemented for {model_type}"
+        )
     if model_type not in MODEL_EXPORTERS:
-        raise ValueError(f"Exporter not implemented for {model_type}")
-
+        raise ValueError(
+            f"Export to Transformers format not implemented for {model_type}"
+        )
+    # Get config
     get_config_fn = MODEL_CONFIGS[model_type]
     hf_config = get_config_fn(backbone)
-
+    # Get weights
     get_weights_fn = MODEL_EXPORTERS[model_type]
-    weights_dict = get_weights_fn(backbone)
-
+    weights_dict = get_weights_fn(backbone, include_lm_head=include_lm_head)
     if not weights_dict:
         raise ValueError("No weights to save.")
-
     # Save config
     os.makedirs(path, exist_ok=True)
     config_path = os.path.join(path, "config.json")
     with open(config_path, "w") as f:
         json.dump(hf_config, f)
-
     # Save weights based on backend
     weights_path = os.path.join(path, "model.safetensors")
     if backend == "torch":
@@ -81,9 +82,28 @@ def export_to_safetensors(keras_model, path):
     else:
         raise ValueError(f"Unsupported backend: {backend}")
 
-    # Save tokenizer assets
-    keras_model.preprocessor.tokenizer.save_assets(path)
 
+def export_tokenizer(tokenizer, path):
+    """Export only the tokenizer to HuggingFace Transformers format.
+
+    Args:
+        tokenizer: The Keras tokenizer to convert.
+        path: str. Path to save the exported tokenizer.
+    """
+    os.makedirs(path, exist_ok=True)
+    # Save tokenizer assets
+    tokenizer.save_assets(path)
+    # Export tokenizer config
+    tokenizer_type = tokenizer.__class__.__name__
+    if tokenizer_type not in MODEL_TOKENIZER_CONFIGS:
+        raise ValueError(
+            "Export to Transformers format not implemented for {tokenizer_type}"
+        )
+    get_tokenizer_config_fn = MODEL_TOKENIZER_CONFIGS[tokenizer_type]
+    tokenizer_config = get_tokenizer_config_fn(tokenizer)
+    tokenizer_config_path = os.path.join(path, "tokenizer_config.json")
+    with open(tokenizer_config_path, "w") as f:
+        json.dump(tokenizer_config, f, indent=4)
     # Rename vocabulary file
     vocab_spm_path = os.path.join(path, "vocabulary.spm")
     tokenizer_model_path = os.path.join(path, "tokenizer.model")
@@ -96,3 +116,29 @@ def export_to_safetensors(keras_model, path):
             "is correct and that the vocabulary file is present "
             "in the original model."
         )
+
+
+def export_to_safetensors(keras_model, path):
+    """Converts a Keras model to Hugging Face Transformers format.
+
+    It does the following:
+    - Exports the backbone (config and weights).
+    - Exports the tokenizer assets.
+
+    Args:
+        keras_model: The Keras model to convert.
+        path: str. Path of the directory to which the safetensors file,
+          config and tokenizer will be saved.
+    """
+    backbone = keras_model.backbone
+    export_backbone(backbone, path, include_lm_head=True)
+    if (
+        keras_model.preprocessor is not None
+        and keras_model.preprocessor.tokenizer is None
+    ):
+        raise ValueError(
+            "CausalLM preprocessor must have a tokenizer for export "
+            "if attached."
+        )
+    if keras_model.preprocessor is not None:
+        export_tokenizer(keras_model.preprocessor.tokenizer, path)

keras_hub/src/version.py

@@ -1,7 +1,7 @@
 from keras_hub.src.api_export import keras_hub_export
 
 # Unique source of truth for the version number.
-__version__ = "0.22.0.dev202508190411"
+__version__ = "0.23.0.dev202508200409"
 
 
 @keras_hub_export("keras_hub.version")

{keras_hub_nightly-0.22.0.dev202508190411.dist-info → keras_hub_nightly-0.23.0.dev202508200409.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: keras-hub-nightly
-Version: 0.22.0.dev202508190411
+Version: 0.23.0.dev202508200409
 Summary: Pretrained models for Keras.
 Author-email: Keras team <keras-users@googlegroups.com>
 License-Expression: Apache-2.0

{keras_hub_nightly-0.22.0.dev202508190411.dist-info → keras_hub_nightly-0.23.0.dev202508200409.dist-info}/RECORD

@@ -5,7 +5,7 @@ keras_hub/models/__init__.py,sha256=UXMwKVZ7bg-AOrq2xsl8M0idUAS89pkdCvQKhzL-D3I,
 keras_hub/samplers/__init__.py,sha256=aFQIkiqbZpi8vjrPp2MVII4QUfE-eQjra5fMeHsoy7k,886
 keras_hub/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/api_export.py,sha256=9pQZK27JObxWZ96QPLBp1OBsjWigh1iuV6RglPGMRk0,1499
-keras_hub/src/version.py,sha256=Ey-gc5DdV3BKvC-FIM1hDevCrbEYvV5XQgYnIzimtds,222
+keras_hub/src/version.py,sha256=PDBU1UqKx4PB-kMmTU1MRLk90YLp6I_7y6FpUU9xYA4,222
 keras_hub/src/layers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/layers/modeling/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/layers/modeling/alibi_bias.py,sha256=1XBTHI52L_iJDhN_w5ydu_iMhCuTgQAxEPwcLA6BPuk,4411
@@ -43,9 +43,9 @@ keras_hub/src/metrics/rouge_n.py,sha256=JoFtmgjF4Ic263ny6bfD6vMHKreH9le3HnOOxemu
 keras_hub/src/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/models/audio_to_text.py,sha256=XoOjXtKBX6K1fz-zOXcdVo3FpjuxCMnJZh2LQcYXb_0,2726
 keras_hub/src/models/audio_to_text_preprocessor.py,sha256=GS-WWyJ6aSsPRxi_0bxvxA00h2mT2FEwSdAoQXAUYVI,3249
-keras_hub/src/models/backbone.py,sha256=utZP09_u5FpMGiq8jl3W98TCW8CysndwLw2VCs3BHz8,11780
-keras_hub/src/models/causal_lm.py,sha256=ReaF-i3SHsCkHh4c28jM72QjMQ8x7yiCwG39FRb-7KE,16786
-keras_hub/src/models/causal_lm_preprocessor.py,sha256=YY7VJZicdmnjDSWi9g4_pEpd5bdJK166GlWcapvokF0,6663
+keras_hub/src/models/backbone.py,sha256=MYc9rJbFRM1q3asI3ORb1sFakV9nBACSQFHnkZBc75M,12323
+keras_hub/src/models/causal_lm.py,sha256=iyPfYhfvM9Rqyc-SZg132KsCYA3Poy-9RRQXN9U8lpE,17671
+keras_hub/src/models/causal_lm_preprocessor.py,sha256=nxl-sfmCfkfl6JmVRASa878QbaZUgWSA6Jdu48x4-dY,7155
 keras_hub/src/models/feature_pyramid_backbone.py,sha256=clEW-TTQSVJ_5qFNdDF0iABkin1p_xlBUFjJrC7T0IA,2247
 keras_hub/src/models/image_classifier.py,sha256=yt6cjhPfqs8A_eWXBsXdXFzn-aRgH2rVHUq7Zu7CyK8,7804
 keras_hub/src/models/image_classifier_preprocessor.py,sha256=Bf7jSqHB1hX2ZWoWQS4GcXNOY_EjeoJi-_vtzCAqw4o,2690
@@ -500,7 +500,7 @@ keras_hub/src/tokenizers/byte_pair_tokenizer.py,sha256=WeUlHMAf5y_MUjFIfVhEcFoOZ
 keras_hub/src/tokenizers/byte_tokenizer.py,sha256=GPIKaddXugbfckfhodADsBpaYb72DgFMs_xfXHnK4qU,10418
 keras_hub/src/tokenizers/sentence_piece_tokenizer.py,sha256=nOqkpa2nHitITpdowPHdwxiN87e8huLW8Dt2gozVnhI,9350
 keras_hub/src/tokenizers/sentence_piece_tokenizer_trainer.py,sha256=caqgV9N4lH97zBviFPdpwo_O95AaJBEJLQv6Icq3Hs8,4774
-keras_hub/src/tokenizers/tokenizer.py,sha256=v0Ka5ayrBwpsGBlkIadXK-b4RsMTbhV6BZrvKullbxY,9722
+keras_hub/src/tokenizers/tokenizer.py,sha256=dk7ZY08k-YMb25FmeK6SQouLKXOvHQjOww2Z30KRyt0,10173
 keras_hub/src/tokenizers/unicode_codepoint_tokenizer.py,sha256=hRv_XxoPIPDpHfO0ZttSOv_M89sMaFpvmllojvKz_ac,13553
 keras_hub/src/tokenizers/word_piece_tokenizer.py,sha256=vP6AZgbzsRiuPCt3W_n94nsF7XiERnagWcH_rqJHtVU,19943
 keras_hub/src/tokenizers/word_piece_tokenizer_trainer.py,sha256=cylrs02ZrYQ1TuZr9oyS3NrVbDwGctA3VXbIh1pFJMQ,6743
@@ -542,11 +542,11 @@ keras_hub/src/utils/transformers/convert_qwen_moe.py,sha256=a7R28aln-PdAcNuKAXdr
 keras_hub/src/utils/transformers/convert_vit.py,sha256=9SUZ9utNJhW_5cj3acMn9cRy47u2eIcDsrhmzj77o9k,5187
 keras_hub/src/utils/transformers/preset_loader.py,sha256=YZVpeNhFITHdauY3MWESrZLNUIJt9ilHJ1jUhvITNT8,4781
 keras_hub/src/utils/transformers/safetensor_utils.py,sha256=CYUHyA4y-B61r7NDnCsFb4t_UmSwZ1k9L-8gzEd6KRg,3339
-keras_hub/src/utils/transformers/export/gemma.py,sha256=NpTSgRUSWp3WXQil1CjYUVFVyyVhpO-4-3q2en2Wxwg,3264
-keras_hub/src/utils/transformers/export/hf_exporter.py,sha256=oTdRS8SalPCbi_cZPC55aZUBc-1_pdviUIp0XysA4cI,3234
+keras_hub/src/utils/transformers/export/gemma.py,sha256=xX_vfQwvFZ_-lQX4kgMNOGKL7fL_1yk7QyGYV2Qyly4,4699
+keras_hub/src/utils/transformers/export/hf_exporter.py,sha256=Qk52c6LIA2eMHUNY9Vy4STJSpnhLMdJ_t-3ljqhSr4k,5081
 keras_hub/tokenizers/__init__.py,sha256=gQIESc4erRLuwxHyxtYy_Z0ePQXw_uhXAa4GVHMffYk,4244
 keras_hub/utils/__init__.py,sha256=jXPqVGBpJr_PpYmqD8aDG-fRMlxH-ulqCR2SZMn288Y,646
-keras_hub_nightly-0.22.0.dev202508190411.dist-info/METADATA,sha256=Rzvs3Xi8nPtcH71ZHF18q0x0yqSuotTblSrBfMjypmg,7395
-keras_hub_nightly-0.22.0.dev202508190411.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-keras_hub_nightly-0.22.0.dev202508190411.dist-info/top_level.txt,sha256=N4J6piIWBKa38A4uV-CnIopnOEf8mHAbkNXafXm_CuA,10
-keras_hub_nightly-0.22.0.dev202508190411.dist-info/RECORD,,
+keras_hub_nightly-0.23.0.dev202508200409.dist-info/METADATA,sha256=vaEex836rRGrO23Lmx6CugNTCP20DYdh6QbntxEIEWg,7395
+keras_hub_nightly-0.23.0.dev202508200409.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+keras_hub_nightly-0.23.0.dev202508200409.dist-info/top_level.txt,sha256=N4J6piIWBKa38A4uV-CnIopnOEf8mHAbkNXafXm_CuA,10
+keras_hub_nightly-0.23.0.dev202508200409.dist-info/RECORD,,