keras-hub-nightly 0.22.0.dev202508190411__tar.gz → 0.23.0.dev202508200409__tar.gz

{keras_hub_nightly-0.22.0.dev202508190411 → keras_hub_nightly-0.23.0.dev202508200409}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: keras-hub-nightly
-Version: 0.22.0.dev202508190411
+Version: 0.23.0.dev202508200409
 Summary: Pretrained models for Keras.
 Author-email: Keras team <keras-users@googlegroups.com>
 License-Expression: Apache-2.0

{keras_hub_nightly-0.22.0.dev202508190411 → keras_hub_nightly-0.23.0.dev202508200409}/keras_hub/src/models/backbone.py

@@ -277,3 +277,19 @@ class Backbone(keras.Model):
             layer.lora_kernel_a.assign(lora_kernel_a)
             layer.lora_kernel_b.assign(lora_kernel_b)
         store.close()
+
+    def export_to_transformers(self, path):
+        """Export the backbone model to HuggingFace Transformers format.
+
+        This saves the backbone's configuration and weights in a format
+        compatible with HuggingFace Transformers. For unsupported model
+        architectures, a ValueError is raised.
+
+        Args:
+            path: str. Path to save the exported model.
+        """
+        from keras_hub.src.utils.transformers.export.hf_exporter import (
+            export_backbone,
+        )
+
+        export_backbone(self, path)

{keras_hub_nightly-0.22.0.dev202508190411 → keras_hub_nightly-0.23.0.dev202508200409}/keras_hub/src/models/causal_lm.py

@@ -392,3 +392,24 @@ class CausalLM(Task):
             outputs = [postprocess(x) for x in outputs]
 
         return self._normalize_generate_outputs(outputs, input_is_scalar)
+
+    def export_to_transformers(self, path):
+        """Export the full CausalLM model to HuggingFace Transformers format.
+
+        This exports the trainable model, tokenizer, and configurations in a
+        format compatible with HuggingFace Transformers. For unsupported model
+        architectures, a ValueError is raised.
+
+        If the preprocessor is attached (default), both the trainable model and
+        tokenizer are exported. To export only the trainable model, set
+        `self.preprocessor = None` before calling this method, then export the
+        preprocessor separately via `preprocessor.export_to_transformers(path)`.
+
+        Args:
+            path: str. Path to save the exported model.
+        """
+        from keras_hub.src.utils.transformers.export.hf_exporter import (
+            export_to_safetensors,
+        )
+
+        export_to_safetensors(self, path)

{keras_hub_nightly-0.22.0.dev202508190411 → keras_hub_nightly-0.23.0.dev202508200409}/keras_hub/src/models/causal_lm_preprocessor.py

@@ -180,3 +180,17 @@ class CausalLMPreprocessor(Preprocessor):
         self._sequence_length = value
         if self.packer is not None:
             self.packer.sequence_length = value
+
+    def export_to_transformers(self, path):
+        """Export the preprocessor to HuggingFace Transformers format.
+
+        Args:
+            path: str. Path to save the exported preprocessor/tokenizer.
+        """
+        if self.tokenizer is None:
+            raise ValueError("Preprocessor must have a tokenizer for export.")
+        from keras_hub.src.utils.transformers.export.hf_exporter import (
+            export_tokenizer,
+        )
+
+        export_tokenizer(self.tokenizer, path)

{keras_hub_nightly-0.22.0.dev202508190411 → keras_hub_nightly-0.23.0.dev202508200409}/keras_hub/src/tokenizers/tokenizer.py

@@ -261,3 +261,18 @@ class Tokenizer(PreprocessingLayer):
         if cls.backbone_cls != backbone_cls:
             cls = find_subclass(preset, cls, backbone_cls)
         return loader.load_tokenizer(cls, config_file, **kwargs)
+
+    def export_to_transformers(self, path):
+        """Export the tokenizer to HuggingFace Transformers format.
+
+        This saves tokenizer assets in a format compatible with HuggingFace
+        Transformers.
+
+        Args:
+            path: str. Path to save the exported tokenizer.
+        """
+        from keras_hub.src.utils.transformers.export.hf_exporter import (
+            export_tokenizer,
+        )
+
+        export_tokenizer(self, path)

{keras_hub_nightly-0.22.0.dev202508190411 → keras_hub_nightly-0.23.0.dev202508200409}/keras_hub/src/utils/transformers/export/gemma.py

@@ -2,6 +2,7 @@ import keras.ops as ops
 
 
 def get_gemma_config(backbone):
+    token_embedding_layer = backbone.get_layer("token_embedding")
     hf_config = {
         "vocab_size": backbone.vocabulary_size,
         "num_hidden_layers": backbone.num_layers,
@@ -11,11 +12,16 @@ def get_gemma_config(backbone):
         "intermediate_size": backbone.intermediate_dim // 2,
         "head_dim": backbone.head_dim,
         "max_position_embeddings": 8192,
+        "tie_word_embeddings": token_embedding_layer.tie_weights,
+        "pad_token_id": 0,
+        "bos_token_id": 2,
+        "eos_token_id": 1,
+        "model_type": "gemma",
     }
     return hf_config
 
 
-def get_gemma_weights_map(backbone):
+def get_gemma_weights_map(backbone, include_lm_head=False):
     weights_dict = {}
 
     # Map token embedding
@@ -83,7 +89,46 @@ def get_gemma_weights_map(backbone):
         "final_normalization"
     ).weights[0]
 
-    # Tie weights, but clone to avoid sharing memory issues
-    weights_dict["lm_head.weight"] = ops.copy(token_embedding_layer.weights[0])
-
+    # Map lm_head if embeddings are not tied
+    if include_lm_head and not token_embedding_layer.tie_weights:
+        weights_dict["lm_head.weight"] = ops.transpose(
+            token_embedding_layer.reverse_embeddings
+        )
     return weights_dict
+
+
+def get_gemma_tokenizer_config(tokenizer):
+    tokenizer_config = {
+        "tokenizer_class": "GemmaTokenizer",
+        "clean_up_tokenization_spaces": False,
+        "bos_token": "<bos>",
+        "eos_token": "<eos>",
+        "pad_token": "<pad>",
+        "unk_token": "<unk>",
+        "add_bos_token": True,
+        "add_eos_token": False,
+        "model_max_length": 8192,
+    }
+    # Add added_tokens_decoder
+    added_tokens_decoder = {}
+    special_tokens = [
+        "<pad>",
+        "<bos>",
+        "<eos>",
+        "<unk>",
+        "<start_of_turn>",
+        "<end_of_turn>",
+    ]
+    for token in special_tokens:
+        token_id = tokenizer.token_to_id(token)
+        if token_id is not None:
+            added_tokens_decoder[str(token_id)] = {
+                "content": token,
+                "special": True,
+                "single_word": False,
+                "lstrip": False,
+                "rstrip": False,
+                "normalized": False,
+            }
+    tokenizer_config["added_tokens_decoder"] = added_tokens_decoder
+    return tokenizer_config

{keras_hub_nightly-0.22.0.dev202508190411 → keras_hub_nightly-0.23.0.dev202508200409}/keras_hub/src/utils/transformers/export/hf_exporter.py

@@ -6,58 +6,59 @@ import warnings
 import keras
 
 from keras_hub.src.utils.transformers.export.gemma import get_gemma_config
+from keras_hub.src.utils.transformers.export.gemma import (
+    get_gemma_tokenizer_config,
+)
 from keras_hub.src.utils.transformers.export.gemma import get_gemma_weights_map
 
 MODEL_CONFIGS = {
     "GemmaBackbone": get_gemma_config,
-    # Add future models here, e.g., "LlamaBackbone": get_llama_config,
+    # Add for future models, e.g., "MistralBackbone": get_mistral_config
 }
 
 MODEL_EXPORTERS = {
     "GemmaBackbone": get_gemma_weights_map,
-    # Add future models here, e.g., "LlamaBackbone": get_llama_weights_map,
+    # Add for future models, e.g., "MistralBackbone": get_mistral_weights_map
 }
 
+MODEL_TOKENIZER_CONFIGS = {
+    "GemmaTokenizer": get_gemma_tokenizer_config,
+    # Add for future models, e.g., "MistralTokenizer":
+    # get_mistral_tokenizer_config
+}
 
-def export_to_safetensors(keras_model, path):
-    """Converts a Keras model to Hugging Face safetensor format.
 
-    It does the following:
-    - Extracts and maps weights from the Keras backbone to safetensors.
-    - Saves the configuration as 'config.json'.
-    - Saves weights in 'model.safetensors'.
-    - Saves tokenizer assets.
+def export_backbone(backbone, path, include_lm_head=False):
+    """Export the backbone model to HuggingFace format.
 
     Args:
-        keras_model: The Keras model to convert.
-        path: str. Path of the directory to which the safetensors file,
-          config and tokenizer will be saved.
+        backbone: The Keras backbone model to convert.
+        path: str. Path to save the exported model.
+        include_lm_head: bool. If True, include lm_head weights if applicable.
     """
     backend = keras.config.backend()
-    backbone = keras_model.backbone
     model_type = backbone.__class__.__name__
-
     if model_type not in MODEL_CONFIGS:
-        raise ValueError(f"Config not implemented for {model_type}")
-
+        raise ValueError(
+            f"Export to Transformers format not implemented for {model_type}"
+        )
     if model_type not in MODEL_EXPORTERS:
-        raise ValueError(f"Exporter not implemented for {model_type}")
-
+        raise ValueError(
+            f"Export to Transformers format not implemented for {model_type}"
+        )
+    # Get config
     get_config_fn = MODEL_CONFIGS[model_type]
     hf_config = get_config_fn(backbone)
-
+    # Get weights
     get_weights_fn = MODEL_EXPORTERS[model_type]
-    weights_dict = get_weights_fn(backbone)
-
+    weights_dict = get_weights_fn(backbone, include_lm_head=include_lm_head)
     if not weights_dict:
         raise ValueError("No weights to save.")
-
     # Save config
     os.makedirs(path, exist_ok=True)
     config_path = os.path.join(path, "config.json")
     with open(config_path, "w") as f:
         json.dump(hf_config, f)
-
     # Save weights based on backend
     weights_path = os.path.join(path, "model.safetensors")
     if backend == "torch":
@@ -81,9 +82,28 @@ def export_to_safetensors(keras_model, path):
     else:
         raise ValueError(f"Unsupported backend: {backend}")
 
-    # Save tokenizer assets
-    keras_model.preprocessor.tokenizer.save_assets(path)
 
+def export_tokenizer(tokenizer, path):
+    """Export only the tokenizer to HuggingFace Transformers format.
+
+    Args:
+        tokenizer: The Keras tokenizer to convert.
+        path: str. Path to save the exported tokenizer.
+    """
+    os.makedirs(path, exist_ok=True)
+    # Save tokenizer assets
+    tokenizer.save_assets(path)
+    # Export tokenizer config
+    tokenizer_type = tokenizer.__class__.__name__
+    if tokenizer_type not in MODEL_TOKENIZER_CONFIGS:
+        raise ValueError(
+            "Export to Transformers format not implemented for {tokenizer_type}"
+        )
+    get_tokenizer_config_fn = MODEL_TOKENIZER_CONFIGS[tokenizer_type]
+    tokenizer_config = get_tokenizer_config_fn(tokenizer)
+    tokenizer_config_path = os.path.join(path, "tokenizer_config.json")
+    with open(tokenizer_config_path, "w") as f:
+        json.dump(tokenizer_config, f, indent=4)
     # Rename vocabulary file
     vocab_spm_path = os.path.join(path, "vocabulary.spm")
     tokenizer_model_path = os.path.join(path, "tokenizer.model")
@@ -96,3 +116,29 @@ def export_to_safetensors(keras_model, path):
             "is correct and that the vocabulary file is present "
             "in the original model."
         )
+
+
+def export_to_safetensors(keras_model, path):
+    """Converts a Keras model to Hugging Face Transformers format.
+
+    It does the following:
+    - Exports the backbone (config and weights).
+    - Exports the tokenizer assets.
+
+    Args:
+        keras_model: The Keras model to convert.
+        path: str. Path of the directory to which the safetensors file,
+          config and tokenizer will be saved.
+    """
+    backbone = keras_model.backbone
+    export_backbone(backbone, path, include_lm_head=True)
+    if (
+        keras_model.preprocessor is not None
+        and keras_model.preprocessor.tokenizer is None
+    ):
+        raise ValueError(
+            "CausalLM preprocessor must have a tokenizer for export "
+            "if attached."
+        )
+    if keras_model.preprocessor is not None:
+        export_tokenizer(keras_model.preprocessor.tokenizer, path)

{keras_hub_nightly-0.22.0.dev202508190411 → keras_hub_nightly-0.23.0.dev202508200409}/keras_hub/src/version.py

@@ -1,7 +1,7 @@
 from keras_hub.src.api_export import keras_hub_export
 
 # Unique source of truth for the version number.
-__version__ = "0.22.0.dev202508190411"
+__version__ = "0.23.0.dev202508200409"
 
 
 @keras_hub_export("keras_hub.version")

{keras_hub_nightly-0.22.0.dev202508190411 → keras_hub_nightly-0.23.0.dev202508200409}/keras_hub_nightly.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: keras-hub-nightly
-Version: 0.22.0.dev202508190411
+Version: 0.23.0.dev202508200409
 Summary: Pretrained models for Keras.
 Author-email: Keras team <keras-users@googlegroups.com>
 License-Expression: Apache-2.0