ai-edge-torch-nightly 0.3.0.dev20240902__py3-none-any.whl → 0.3.0.dev20240905__py3-none-any.whl

ai_edge_torch/generative/examples/experimental/gemma/gemma.py

@@ -21,15 +21,16 @@ import os
 from pathlib import Path
 from typing import Tuple
 
+from ai_edge_torch.generative.layers import builder
 import ai_edge_torch.generative.layers.attention_utils as attn_utils
-import ai_edge_torch.generative.layers.builder as builder
+from ai_edge_torch.generative.layers.experimental import attention
 from ai_edge_torch.generative.layers.experimental import ekv_cache as kv_utils
-from ai_edge_torch.generative.layers.experimental.attention import TransformerBlock  # NOQA
 import ai_edge_torch.generative.layers.model_config as cfg
 import ai_edge_torch.generative.utilities.loader as loading_utils
 import numpy as np
 import torch
-import torch.nn as nn
+from torch import nn
+
 
 TENSOR_NAMES = loading_utils.ModelLoader.TensorNames(
     ff_up_proj="model.layers.{}.mlp.up_proj",
@@ -48,6 +49,7 @@ TENSOR_NAMES = loading_utils.ModelLoader.TensorNames(
 
 
 class Gemma(nn.Module):
+  """A Gemma model built from the Edge Generative API layers."""
 
   def __init__(self, config: cfg.ModelConfig):
     super().__init__()
@@ -65,7 +67,7 @@ class Gemma(nn.Module):
     # Gemma re-uses the embedding as the head projection layer.
     self.lm_head.weight.data = self.tok_embedding.weight.data
     self.transformer_blocks = nn.ModuleList(
-        TransformerBlock(config) for _ in range(config.num_layers)
+        attention.TransformerBlock(config) for _ in range(config.num_layers)
     )
     self.final_norm = builder.build_norm(
         config.embedding_dim,
@@ -95,9 +97,9 @@ class Gemma(nn.Module):
       input_pos: torch.Tensor,
       kv_cache: kv_utils.EKVCache,
   ) -> Tuple[torch.Tensor, kv_utils.EKVCache]:
-    B, T = tokens.size()
-    assert self.config.max_seq_len >= T, (
-        f"Cannot forward sequence of length {T}, max seq length is only"
+    _, seq_len = tokens.size()
+    assert self.config.max_seq_len >= seq_len, (
+        f"Cannot forward sequence of length {seq_len}, max seq length is only"
         f" {self.config.max_seq_len}"
     )
 
@@ -125,6 +127,15 @@ class Gemma(nn.Module):
 
 
 def get_model_config_2b(kv_cache_max_len: int = 1024) -> cfg.ModelConfig:
+  """Returns the model config for a Gemma 2B model.
+
+  Args:
+    kv_cache_max_len (int): The maximum sequence length of the KV cache. Default
+      is 1024.
+
+  Returns:
+    The model config for a Gemma 2B model.
+  """
   attn_config = cfg.AttentionConfig(
       num_heads=8,
       head_dim=256,
@@ -160,41 +171,18 @@ def get_model_config_2b(kv_cache_max_len: int = 1024) -> cfg.ModelConfig:
 
 
 def get_fake_model_config(kv_cache_max_len: int = 128) -> cfg.ModelConfig:
-  attn_config = cfg.AttentionConfig(
-      num_heads=8,
-      head_dim=256,
-      num_query_groups=1,
-      rotary_percentage=1.0,
-  )
-  ff_config = cfg.FeedForwardConfig(
-      type=cfg.FeedForwardType.GATED,
-      activation=cfg.ActivationConfig(cfg.ActivationType.GELU_TANH),
-      intermediate_size=128,
-  )
-  norm_config = cfg.NormalizationConfig(
-      type=cfg.NormalizationType.RMS_NORM,
-      epsilon=1e-6,
-      zero_centered=True,
-  )
-  config = cfg.ModelConfig(
-      vocab_size=128,
-      num_layers=2,
-      max_seq_len=2 * kv_cache_max_len,
-      embedding_dim=2048,
-      kv_cache_max_len=kv_cache_max_len,
-      attn_config=attn_config,
-      ff_config=ff_config,
-      pre_attention_norm_config=norm_config,
-      post_attention_norm_config=norm_config,
-      final_norm_config=norm_config,
-      parallel_residual=False,
-      lm_head_use_bias=False,
-      enable_hlfb=True,
-  )
+  config = get_model_config_2b(kv_cache_max_len)
+  config.ff_config.intermediate_size = 128
+  config.vocab_size = 128
+  config.num_layers = 2
+  config.max_seq_len = 2 * kv_cache_max_len
   return config
 
 
-def build_2b_model(checkpoint_path, test_model=False, **kwargs) -> nn.Module:
+def build_2b_model(
+    checkpoint_path: str, test_model: bool = False, **kwargs
+) -> nn.Module:
+  """Instantiates the model instance and load checkpoint if provided."""
   config = (
       get_fake_model_config(**kwargs)
       if test_model
@@ -210,7 +198,9 @@ def build_2b_model(checkpoint_path, test_model=False, **kwargs) -> nn.Module:
   return model
 
 
-def define_and_run_2b(checkpoint_path, test_model=False) -> None:
+def define_and_run_2b(checkpoint_path: str, test_model: bool = False) -> None:
+  """Instantiates and runs a Gemma 2B model."""
+
   kv_cache_max_len = 1024
   model = build_2b_model(
       checkpoint_path, test_model=test_model, kv_cache_max_len=kv_cache_max_len
@@ -225,5 +215,5 @@ def define_and_run_2b(checkpoint_path, test_model=False) -> None:
 
 
 if __name__ == "__main__":
-  checkpoint_path = os.path.join(Path.home(), "Downloads/gemma-2b")
-  define_and_run_2b(checkpoint_path)
+  input_checkpoint_path = os.path.join(Path.home(), "Downloads/gemma-2b")
+  define_and_run_2b(input_checkpoint_path)

ai_edge_torch/generative/examples/experimental/phi/phi2.py

@@ -17,20 +17,20 @@
 # Note: This is an experimental version of phi2 with external KV cache.
 # Please use with caution.
 
-
 import os
 from pathlib import Path
 from typing import Tuple
 
+from ai_edge_torch.generative.layers import builder
 import ai_edge_torch.generative.layers.attention_utils as attn_utils
-import ai_edge_torch.generative.layers.builder as builder
+from ai_edge_torch.generative.layers.experimental import attention
 from ai_edge_torch.generative.layers.experimental import ekv_cache as kv_utils
-from ai_edge_torch.generative.layers.experimental.attention import TransformerBlock  # NOQA
 import ai_edge_torch.generative.layers.model_config as cfg
 import ai_edge_torch.generative.utilities.loader as loading_utils
 import numpy as np
 import torch
-import torch.nn as nn
+from torch import nn
+
 
 TENSOR_NAMES = loading_utils.ModelLoader.TensorNames(
     ff_up_proj="model.layers.{}.mlp.fc1",
@@ -47,6 +47,7 @@ TENSOR_NAMES = loading_utils.ModelLoader.TensorNames(
 
 
 class Phi2(nn.Module):
+  """A Phi-2 model built from the Edge Generative API layers."""
 
   def __init__(self, config: cfg.ModelConfig):
     super().__init__()
@@ -60,7 +61,7 @@ class Phi2(nn.Module):
         config.vocab_size, config.embedding_dim, padding_idx=0
     )
     self.transformer_blocks = nn.ModuleList(
-        TransformerBlock(config) for _ in range(config.num_layers)
+        attention.TransformerBlock(config) for _ in range(config.num_layers)
     )
     self.final_norm = builder.build_norm(
         config.embedding_dim,
@@ -90,9 +91,9 @@ class Phi2(nn.Module):
       input_pos: torch.Tensor,
       kv_cache: kv_utils.EKVCache,
   ) -> Tuple[torch.Tensor, kv_utils.EKVCache]:
-    B, T = tokens.size()
-    assert self.config.max_seq_len >= T, (
-        f"Cannot forward sequence of length {T}, max seq length is only"
+    _, seq_len = tokens.size()
+    assert self.config.max_seq_len >= seq_len, (
+        f"Cannot forward sequence of length {seq_len}, max seq length is only"
         f" {self.config.max_seq_len}"
     )
 
@@ -118,6 +119,15 @@ class Phi2(nn.Module):
 
 
 def get_model_config(kv_cache_max_len: int = 1024) -> cfg.ModelConfig:
+  """Returns the model config for a Phi-2 model.
+
+  Args:
+    kv_cache_max_len (int): The maximum sequence length of the KV cache. Default
+      is 1024.
+
+  Returns:
+    The model config for a Phi-2 model.
+  """
   attn_config = cfg.AttentionConfig(
       num_heads=32,
       head_dim=80,
@@ -150,15 +160,21 @@ def get_model_config(kv_cache_max_len: int = 1024) -> cfg.ModelConfig:
   return config
 
 
-def get_fake_model_config_for_test(**kwargs) -> cfg.ModelConfig:
-  config = get_model_config(**kwargs)
+def get_fake_model_config(kv_cache_max_len: int = 128) -> cfg.ModelConfig:
+  config = get_model_config(kv_cache_max_len)
+  config.vocab_size = 128
   config.num_layers = 2
+  config.max_seq_len = 2 * kv_cache_max_len
+  config.ff_config.intermediate_size = 128
   return config
 
 
-def build_model(checkpoint_path, test_model=False, **kwargs) -> nn.Module:
+def build_model(
+    checkpoint_path: str, test_model: bool = False, **kwargs
+) -> nn.Module:
+  """Instantiates the model instance and load checkpoint if provided."""
   config = (
-      get_fake_model_config_for_test(**kwargs)
+      get_fake_model_config(**kwargs)
       if test_model
       else get_model_config(**kwargs)
   )
@@ -170,7 +186,9 @@ def build_model(checkpoint_path, test_model=False, **kwargs) -> nn.Module:
   return model
 
 
-def define_and_run(checkpoint_path, test_model=False) -> None:
+def define_and_run(checkpoint_path: str, test_model: bool = False) -> None:
+  """Instantiates and runs a Phi-2 model."""
+
   kv_cache_max_len = 1024
   model = build_model(
       checkpoint_path, test_model=test_model, kv_cache_max_len=kv_cache_max_len
@@ -185,5 +203,5 @@ def define_and_run(checkpoint_path, test_model=False) -> None:
 
 
 if __name__ == "__main__":
-  checkpoint_path = os.path.join(Path.home(), "Downloads/phi2")
-  define_and_run(checkpoint_path)
+  input_checkpoint_path = os.path.join(Path.home(), "Downloads/phi2")
+  define_and_run(input_checkpoint_path)

ai_edge_torch/generative/examples/experimental/tiny_llama/tiny_llama.py

@@ -17,20 +17,20 @@
 # Note: This is an experimental version of TinyLlama with external KV cache.
 # Please use with caution.
 
-
 import os
 from pathlib import Path
 from typing import Tuple
 
+from ai_edge_torch.generative.layers import builder
 import ai_edge_torch.generative.layers.attention_utils as attn_utils
-import ai_edge_torch.generative.layers.builder as builder
+from ai_edge_torch.generative.layers.experimental import attention
 from ai_edge_torch.generative.layers.experimental import ekv_cache as kv_utils
-from ai_edge_torch.generative.layers.experimental.attention import TransformerBlock  # NOQA
 import ai_edge_torch.generative.layers.model_config as cfg
 import ai_edge_torch.generative.utilities.loader as loading_utils
 import numpy as np
 import torch
-import torch.nn as nn
+from torch import nn
+
 
 TENSOR_NAMES = loading_utils.ModelLoader.TensorNames(
     ff_up_proj="model.layers.{}.mlp.up_proj",
@@ -49,6 +49,7 @@ TENSOR_NAMES = loading_utils.ModelLoader.TensorNames(
 
 
 class TinyLLamma(nn.Module):
+  """A TinyLlama model built from the Edge Generative API layers."""
 
   def __init__(self, config: cfg.ModelConfig):
     super().__init__()
@@ -62,7 +63,7 @@ class TinyLLamma(nn.Module):
         config.vocab_size, config.embedding_dim, padding_idx=0
     )
     self.transformer_blocks = nn.ModuleList(
-        TransformerBlock(config) for _ in range(config.num_layers)
+        attention.TransformerBlock(config) for _ in range(config.num_layers)
     )
     self.final_norm = builder.build_norm(
         config.embedding_dim,
@@ -92,9 +93,9 @@ class TinyLLamma(nn.Module):
       input_pos: torch.Tensor,
       kv_cache: kv_utils.EKVCache,
   ) -> Tuple[torch.Tensor, kv_utils.EKVCache]:
-    B, T = tokens.size()
-    assert self.config.max_seq_len >= T, (
-        f"Cannot forward sequence of length {T}, max seq length is only"
+    _, seq_len = tokens.size()
+    assert self.config.max_seq_len >= seq_len, (
+        f"Cannot forward sequence of length {seq_len}, max seq length is only"
         f" {self.config.max_seq_len}"
     )
 
@@ -121,6 +122,15 @@ class TinyLLamma(nn.Module):
 
 
 def get_model_config(kv_cache_max_len: int = 1024) -> cfg.ModelConfig:
+  """Returns the model config for a TinyLlama model.
+
+  Args:
+    kv_cache_max_len (int): The maximum sequence length of the KV cache. Default
+      is 1024.
+
+  Returns:
+    The model config for a TinyLlama model.
+  """
   attn_config = cfg.AttentionConfig(
       num_heads=32,
       head_dim=64,
@@ -149,7 +159,7 @@ def get_model_config(kv_cache_max_len: int = 1024) -> cfg.ModelConfig:
   return config
 
 
-def get_fake_model_config_for_test(**kwargs) -> cfg.ModelConfig:
+def get_fake_model_config(**kwargs) -> cfg.ModelConfig:
   config = get_model_config(**kwargs)
   config.vocab_size = 128
   config.num_layers = 2
@@ -157,9 +167,12 @@ def get_fake_model_config_for_test(**kwargs) -> cfg.ModelConfig:
   return config
 
 
-def build_model(checkpoint_path, test_model=False, **kwargs) -> nn.Module:
+def build_model(
+    checkpoint_path: str, test_model: bool = False, **kwargs
+) -> nn.Module:
+  """Instantiates the model instance and load checkpoint if provided."""
   config = (
-      get_fake_model_config_for_test(**kwargs)
+      get_fake_model_config(**kwargs)
       if test_model
       else get_model_config(**kwargs)
   )
@@ -171,7 +184,9 @@ def build_model(checkpoint_path, test_model=False, **kwargs) -> nn.Module:
   return model
 
 
-def define_and_run(checkpoint_path, test_model=False) -> None:
+def define_and_run(checkpoint_path: str, test_model: bool = False) -> None:
+  """Instantiates and runs a TinyLlama model."""
+
   kv_cache_max_len = 1024
   model = build_model(
       checkpoint_path, test_model=test_model, kv_cache_max_len=kv_cache_max_len
@@ -186,5 +201,5 @@ def define_and_run(checkpoint_path, test_model=False) -> None:
 
 
 if __name__ == "__main__":
-  checkpoint_path = os.path.join(Path.home(), "Downloads/tiny_llama")
-  define_and_run(checkpoint_path)
+  input_checkpoint_path = os.path.join(Path.home(), "Downloads/tiny_llama")
+  define_and_run(input_checkpoint_path)

ai_edge_torch/generative/examples/gemma/gemma.py

@@ -17,14 +17,14 @@
 import os
 from pathlib import Path
 
-from ai_edge_torch.generative.layers.attention import TransformerBlock
+from ai_edge_torch.generative.layers import attention
+from ai_edge_torch.generative.layers import builder
 import ai_edge_torch.generative.layers.attention_utils as attn_utils
-import ai_edge_torch.generative.layers.builder as builder
 import ai_edge_torch.generative.layers.model_config as cfg
 import ai_edge_torch.generative.utilities.loader as loading_utils
 import numpy as np
 import torch
-import torch.nn as nn
+from torch import nn
 
 TENSOR_NAMES = loading_utils.ModelLoader.TensorNames(
     ff_up_proj="model.layers.{}.mlp.up_proj",
@@ -43,6 +43,7 @@ TENSOR_NAMES = loading_utils.ModelLoader.TensorNames(
 
 
 class Gemma(nn.Module):
+  """A Gemma model built from the Edge Generative API layers."""
 
   def __init__(self, config: cfg.ModelConfig):
     super().__init__()
@@ -60,7 +61,7 @@ class Gemma(nn.Module):
     # Gemma re-uses the embedding as the head projection layer.
     self.lm_head.weight.data = self.tok_embedding.weight.data
     self.transformer_blocks = nn.ModuleList(
-        TransformerBlock(config) for _ in range(config.num_layers)
+        attention.TransformerBlock(config) for _ in range(config.num_layers)
     )
     self.final_norm = builder.build_norm(
         config.embedding_dim,
@@ -88,9 +89,9 @@ class Gemma(nn.Module):
   # This can be eliminated if we handle k/v cache updates inside the model itself.
   @torch.inference_mode
   def forward(self, idx: torch.Tensor, input_pos: torch.Tensor) -> torch.Tensor:
-    B, T = idx.size()
-    assert self.config.max_seq_len >= T, (
-        f"Cannot forward sequence of length {T}, max seq length is only"
+    _, seq_len = idx.size()
+    assert self.config.max_seq_len >= seq_len, (
+        f"Cannot forward sequence of length {seq_len}, max seq length is only"
         f" {self.config.max_seq_len}"
     )
 
@@ -104,7 +105,7 @@ class Gemma(nn.Module):
     x = self.tok_embedding(idx)
     x = x * (self.config.embedding_dim**0.5)
 
-    for i, block in enumerate(self.transformer_blocks):
+    for _, block in enumerate(self.transformer_blocks):
       x = block(x, (cos, sin), mask, input_pos)
 
     x = self.final_norm(x)
@@ -113,6 +114,15 @@ class Gemma(nn.Module):
 
 
 def get_model_config_2b(kv_cache_max_len: int = 1024) -> cfg.ModelConfig:
+  """Returns the model config for a Gemma 2B model.
+
+  Args:
+    kv_cache_max_len (int): The maximum sequence length of the KV cache. Default
+      is 1024.
+
+  Returns:
+    The model config for a Gemma 2B model.
+  """
   attn_config = cfg.AttentionConfig(
       num_heads=8,
       head_dim=256,
@@ -147,43 +157,16 @@ def get_model_config_2b(kv_cache_max_len: int = 1024) -> cfg.ModelConfig:
   return config
 
 
-# TODO(b/363021962): Clean up this part to streamline fake model config generation.
 def get_fake_model_config(kv_cache_max_len: int = 128) -> cfg.ModelConfig:
-  attn_config = cfg.AttentionConfig(
-      num_heads=8,
-      head_dim=256,
-      num_query_groups=1,
-      rotary_percentage=1.0,
-  )
-  ff_config = cfg.FeedForwardConfig(
-      type=cfg.FeedForwardType.GATED,
-      activation=cfg.ActivationConfig(cfg.ActivationType.GELU_TANH),
-      intermediate_size=128,
-  )
-  norm_config = cfg.NormalizationConfig(
-      type=cfg.NormalizationType.RMS_NORM,
-      epsilon=1e-6,
-      zero_centered=True,
-  )
-  config = cfg.ModelConfig(
-      vocab_size=128,
-      num_layers=2,
-      max_seq_len=2 * kv_cache_max_len,
-      embedding_dim=2048,
-      kv_cache_max_len=kv_cache_max_len,
-      attn_config=attn_config,
-      ff_config=ff_config,
-      pre_attention_norm_config=norm_config,
-      post_attention_norm_config=norm_config,
-      final_norm_config=norm_config,
-      parallel_residual=False,
-      lm_head_use_bias=False,
-      enable_hlfb=True,
-  )
+  config = get_model_config_2b(kv_cache_max_len)
+  config.ff_config.intermediate_size = 128
+  config.vocab_size = 128
+  config.num_layers = 2
+  config.max_seq_len = 2 * kv_cache_max_len
   return config
 
 
-def build_2b_model(checkpoint_path, **kwargs) -> nn.Module:
+def build_2b_model(checkpoint_path: str, **kwargs) -> nn.Module:
   config = get_model_config_2b(**kwargs)
   model = Gemma(config)
   loader = loading_utils.ModelLoader(checkpoint_path, TENSOR_NAMES)
@@ -195,6 +178,8 @@ def build_2b_model(checkpoint_path, **kwargs) -> nn.Module:
 
 
 def define_and_run_2b() -> None:
+  """Instantiates and runs a Gemma 2B model."""
+
   current_dir = Path(__file__).parent.resolve()
   gemma_goldens = torch.load(current_dir / "gemma_lm_logits.pt")
 

ai_edge_torch/generative/examples/gemma/gemma2.py

@@ -18,14 +18,14 @@ import os
 from pathlib import Path
 from typing import Optional, Tuple
 
-from ai_edge_torch.generative.layers.attention import TransformerBlock
+from ai_edge_torch.generative.layers import attention
+from ai_edge_torch.generative.layers import builder
 import ai_edge_torch.generative.layers.attention_utils as attn_utils
-import ai_edge_torch.generative.layers.builder as builder
 import ai_edge_torch.generative.layers.model_config as cfg
 import ai_edge_torch.generative.utilities.loader as loading_utils
 import numpy as np
 import torch
-import torch.nn as nn
+from torch import nn
 
 TENSOR_NAMES = loading_utils.ModelLoader.TensorNames(
     ff_up_proj="model.layers.{}.mlp.up_proj",
@@ -43,7 +43,7 @@ TENSOR_NAMES = loading_utils.ModelLoader.TensorNames(
 )
 
 
-class Gemma2Block(TransformerBlock):
+class Gemma2Block(attention.TransformerBlock):
 
   def forward(
       self,
@@ -76,6 +76,7 @@ class Gemma2Block(TransformerBlock):
 
 
 class Gemma2(nn.Module):
+  """A Gemma2 model built from the Edge Generative API layers."""
 
   def __init__(self, config: cfg.ModelConfig):
     super().__init__()
@@ -138,9 +139,9 @@ class Gemma2(nn.Module):
 
   @torch.inference_mode
   def forward(self, idx: torch.Tensor, input_pos: torch.Tensor) -> torch.Tensor:
-    B, T = idx.size()
-    assert self.config.max_seq_len >= T, (
-        f"Cannot forward sequence of length {T}, max seq length is only"
+    _, seq_len = idx.size()
+    assert self.config.max_seq_len >= seq_len, (
+        f"Cannot forward sequence of length {seq_len}, max seq length is only"
         f" {self.config.max_seq_len}"
     )
 
@@ -166,6 +167,15 @@ class Gemma2(nn.Module):
 
 
 def get_model_config_2b(kv_cache_max_len: int = 1024) -> cfg.ModelConfig:
+  """Returns the model config for a Gemma2 2B model.
+
+  Args:
+    kv_cache_max_len (int): The maximum sequence length of the KV cache. Default
+      is 1024.
+
+  Returns:
+    The model config for a Gemma 2B model.
+  """
   attn_config = cfg.AttentionConfig(
       num_heads=8,
       head_dim=256,
@@ -210,50 +220,19 @@ def get_model_config_2b(kv_cache_max_len: int = 1024) -> cfg.ModelConfig:
 
 
 def get_fake_model_config(kv_cache_max_len: int = 128) -> cfg.ModelConfig:
-  attn_config = cfg.AttentionConfig(
-      num_heads=4,
-      head_dim=64,
-      num_query_groups=4,
-      rotary_percentage=1.0,
-      qkv_transpose_before_split=True,
-      logit_softcap=50.0,
-      sliding_window_size=64,
-      attn_types=[cfg.AttentionType.GLOBAL, cfg.AttentionType.LOCAL_SLIDING]
-      * 13,
-  )
-
-  norm_config = cfg.NormalizationConfig(
-      type=cfg.NormalizationType.RMS_NORM,
-      epsilon=1e-6,
-      zero_centered=True,
-  )
-  ff_config = cfg.FeedForwardConfig(
-      type=cfg.FeedForwardType.GATED,
-      activation=cfg.ActivationConfig(cfg.ActivationType.GELU_TANH),
-      intermediate_size=128,
-      pre_ff_norm_config=norm_config,
-      post_ff_norm_config=norm_config,
-  )
-  config = cfg.ModelConfig(
-      vocab_size=128,
-      num_layers=2,
-      max_seq_len=2 * kv_cache_max_len,
-      embedding_dim=128,
-      kv_cache_max_len=kv_cache_max_len,
-      attn_config=attn_config,
-      ff_config=ff_config,
-      pre_attention_norm_config=norm_config,
-      post_attention_norm_config=norm_config,
-      final_norm_config=norm_config,
-      parallel_residual=False,
-      lm_head_use_bias=False,
-      enable_hlfb=True,
-      final_logit_softcap=30.0,
-  )
+  config = get_model_config_2b(kv_cache_max_len)
+  config.attn_config.num_heads = 4
+  config.attn_config.head_dim = 64
+  config.attn_config.sliding_window_size = 64
+  config.ff_config.intermediate_size = 128
+  config.vocab_size = 128
+  config.num_layers = 2
+  config.max_seq_len = 2 * kv_cache_max_len
+  config.embedding_dim = 128
   return config
 
 
-def build_2b_model(checkpoint_path, **kwargs) -> nn.Module:
+def build_2b_model(checkpoint_path: str, **kwargs) -> nn.Module:
   config = get_model_config_2b(**kwargs)
   model = Gemma2(config)
   loader = loading_utils.ModelLoader(checkpoint_path, TENSOR_NAMES)
@@ -265,6 +244,8 @@ def build_2b_model(checkpoint_path, **kwargs) -> nn.Module:
 
 
 def define_and_run_2b() -> None:
+  """Instantiates and runs a Gemma2 2B model."""
+
   current_dir = Path(__file__).parent.resolve()
   gemma2_goldens = torch.load(current_dir / "gemma2it_2b_golden.pt")
   print("Running GEMMA 2")

ai_edge_torch/generative/examples/phi2/phi2.py

@@ -18,14 +18,14 @@
 import os
 from pathlib import Path
 
-from ai_edge_torch.generative.layers.attention import TransformerBlock
+from ai_edge_torch.generative.layers import attention
+from ai_edge_torch.generative.layers import builder
 import ai_edge_torch.generative.layers.attention_utils as attn_utils
-import ai_edge_torch.generative.layers.builder as builder
 import ai_edge_torch.generative.layers.model_config as cfg
 import ai_edge_torch.generative.utilities.loader as loading_utils
 import numpy as np
 import torch
-import torch.nn as nn
+from torch import nn
 
 TENSOR_NAMES = loading_utils.ModelLoader.TensorNames(
     ff_up_proj="model.layers.{}.mlp.fc1",
@@ -42,6 +42,7 @@ TENSOR_NAMES = loading_utils.ModelLoader.TensorNames(
 
 
 class Phi2(nn.Module):
+  """A Phi-2 model built from the Edge Generative API layers."""
 
   def __init__(self, config: cfg.ModelConfig):
     super().__init__()
@@ -55,7 +56,7 @@ class Phi2(nn.Module):
         config.vocab_size, config.embedding_dim, padding_idx=0
     )
     self.transformer_blocks = nn.ModuleList(
-        TransformerBlock(config) for _ in range(config.num_layers)
+        attention.TransformerBlock(config) for _ in range(config.num_layers)
     )
     self.final_norm = builder.build_norm(
         config.embedding_dim,
@@ -83,9 +84,9 @@ class Phi2(nn.Module):
   # This can be eliminated if we handle k/v cache updates inside the model itself.
   @torch.inference_mode
   def forward(self, idx: torch.Tensor, input_pos: torch.Tensor) -> torch.Tensor:
-    B, T = idx.size()
-    assert self.config.max_seq_len >= T, (
-        f"Cannot forward sequence of length {T}, max seq length is only"
+    _, seq_len = idx.size()
+    assert self.config.max_seq_len >= seq_len, (
+        f"Cannot forward sequence of length {seq_len}, max seq length is only"
         f" {self.config.max_seq_len}"
     )
 
@@ -98,7 +99,7 @@ class Phi2(nn.Module):
     # forward the model itself
     x = self.tok_embedding(idx)  # token embeddings of shape (b, t, n_embd)
 
-    for i, block in enumerate(self.transformer_blocks):
+    for _, block in enumerate(self.transformer_blocks):
       x = block(x, (cos, sin), mask, input_pos)
 
     x = self.final_norm(x)
@@ -107,6 +108,15 @@ class Phi2(nn.Module):
 
 
 def get_model_config(kv_cache_max_len: int = 1024) -> cfg.ModelConfig:
+  """Returns the model config for a Phi-2 model.
+
+  Args:
+    kv_cache_max_len (int): The maximum sequence length of the KV cache. Default
+      is 1024.
+
+  Returns:
+    The model config for a Phi-2 model.
+  """
   attn_config = cfg.AttentionConfig(
       num_heads=32,
       head_dim=80,
@@ -140,35 +150,11 @@ def get_model_config(kv_cache_max_len: int = 1024) -> cfg.ModelConfig:
 
 
 def get_fake_model_config(kv_cache_max_len: int = 128) -> cfg.ModelConfig:
-  attn_config = cfg.AttentionConfig(
-      num_heads=16,
-      head_dim=80,
-      num_query_groups=4,
-      rotary_percentage=0.4,
-      qkv_use_bias=True,
-      output_proj_use_bias=True,
-  )
-  ff_config = cfg.FeedForwardConfig(
-      type=cfg.FeedForwardType.SEQUENTIAL,
-      activation=cfg.ActivationConfig(cfg.ActivationType.GELU_TANH),
-      intermediate_size=128,
-      use_bias=True,
-  )
-  norm_config = cfg.NormalizationConfig(type=cfg.NormalizationType.LAYER_NORM)
-  config = cfg.ModelConfig(
-      vocab_size=128,
-      num_layers=2,
-      max_seq_len=2 * kv_cache_max_len,
-      kv_cache_max_len=kv_cache_max_len,
-      embedding_dim=128,
-      attn_config=attn_config,
-      ff_config=ff_config,
-      pre_attention_norm_config=norm_config,
-      final_norm_config=norm_config,
-      parallel_residual=True,
-      lm_head_use_bias=True,
-      enable_hlfb=True,
-  )
+  config = get_model_config(kv_cache_max_len)
+  config.vocab_size = 128
+  config.num_layers = 2
+  config.max_seq_len = 2 * kv_cache_max_len
+  config.ff_config.intermediate_size = 128
   return config
 
 
@@ -181,6 +167,8 @@ def build_model(checkpoint_path, **kwargs) -> nn.Module:
 
 
 def define_and_run() -> None:
+  """Instantiates and runs a Phi-2 model."""
+
   current_dir = Path(__file__).parent.resolve()
   phi2_goldens = torch.load(current_dir / "phi2_lm_logits.pt")
   kv_cache_max_len = 1024

ai_edge_torch/generative/examples/test_models/toy_model.py

@@ -71,6 +71,56 @@ class ToySingleLayerModel(torch.nn.Module):
     return self.lm_head(x)
 
 
+class ToySingleLayerModelWeightSharing(torch.nn.Module):
+
+  def __init__(self, config: cfg.ModelConfig) -> None:
+    super().__init__()
+    self.lm_head = nn.Linear(
+        config.embedding_dim, config.vocab_size, bias=config.lm_head_use_bias
+    )
+    self.tok_embedding = nn.Embedding(config.vocab_size, config.embedding_dim)
+    self.lm_head = nn.Linear(
+        config.embedding_dim,
+        config.vocab_size,
+        bias=config.lm_head_use_bias,
+    )
+    self.lm_head.weight.data = self.tok_embedding.weight.data
+    self.transformer_block = TransformerBlock(config)
+    self.final_norm = builder.build_norm(
+        config.embedding_dim,
+        config.final_norm_config,
+    )
+    self.rope_cache = attn_utils.build_rope_cache(
+        size=config.max_seq_len,
+        dim=int(
+            config.attn_config.rotary_percentage * config.attn_config.head_dim
+        ),
+        base=10_000,
+        condense_ratio=1,
+        dtype=torch.float32,
+        device=torch.device('cpu'),
+    )
+    self.mask_cache = attn_utils.build_causal_mask_cache(
+        size=config.max_seq_len, dtype=torch.float32, device=torch.device('cpu')
+    )
+    self.config = config
+
+  @torch.inference_mode
+  def forward(self, idx: torch.Tensor, input_pos: torch.Tensor) -> torch.Tensor:
+    x = self.tok_embedding(idx)
+    cos, sin = self.rope_cache
+
+    cos = cos.index_select(0, input_pos)
+    sin = sin.index_select(0, input_pos)
+    mask = self.mask_cache.index_select(2, input_pos)
+    mask = mask[:, :, :, : self.config.max_seq_len]
+
+    x = self.transformer_block(x, (cos, sin), mask, input_pos)
+    x = self.final_norm(x)
+    res = self.lm_head(x)
+    return res
+
+
 def get_model_config() -> cfg.ModelConfig:
   attn_config = cfg.AttentionConfig(
       num_heads=32,

ai_edge_torch/generative/examples/tiny_llama/tiny_llama.py

@@ -17,14 +17,14 @@
 import os
 from pathlib import Path
 
-from ai_edge_torch.generative.layers.attention import TransformerBlock
+from ai_edge_torch.generative.layers import attention
+from ai_edge_torch.generative.layers import builder
 import ai_edge_torch.generative.layers.attention_utils as attn_utils
-import ai_edge_torch.generative.layers.builder as builder
 import ai_edge_torch.generative.layers.model_config as cfg
 import ai_edge_torch.generative.utilities.loader as loading_utils
 import numpy as np
 import torch
-import torch.nn as nn
+from torch import nn
 
 TENSOR_NAMES = loading_utils.ModelLoader.TensorNames(
     ff_up_proj="model.layers.{}.mlp.up_proj",
@@ -43,6 +43,7 @@ TENSOR_NAMES = loading_utils.ModelLoader.TensorNames(
 
 
 class TinyLLamma(nn.Module):
+  """A TinyLlama model built from the Edge Generative API layers."""
 
   def __init__(self, config: cfg.ModelConfig):
     super().__init__()
@@ -56,7 +57,7 @@ class TinyLLamma(nn.Module):
         config.vocab_size, config.embedding_dim, padding_idx=0
     )
     self.transformer_blocks = nn.ModuleList(
-        TransformerBlock(config) for _ in range(config.num_layers)
+        attention.TransformerBlock(config) for _ in range(config.num_layers)
     )
     self.final_norm = builder.build_norm(
         config.embedding_dim,
@@ -84,9 +85,9 @@ class TinyLLamma(nn.Module):
   # This can be eliminated if we handle k/v cache updates inside the model itself.
   @torch.inference_mode
   def forward(self, idx: torch.Tensor, input_pos: torch.Tensor) -> torch.Tensor:
-    B, T = idx.size()
-    assert self.config.max_seq_len >= T, (
-        f"Cannot forward sequence of length {T}, max seq length is only"
+    _, seq_len = idx.size()
+    assert self.config.max_seq_len >= seq_len, (
+        f"Cannot forward sequence of length {seq_len}, max seq length is only"
         f" {self.config.max_seq_len}"
     )
 
@@ -99,7 +100,7 @@ class TinyLLamma(nn.Module):
     # forward the model itself
     x = self.tok_embedding(idx)  # token embeddings of shape (b, t, n_embd)
 
-    for i, block in enumerate(self.transformer_blocks):
+    for _, block in enumerate(self.transformer_blocks):
       x = block(x, (cos, sin), mask, input_pos)
 
     x = self.final_norm(x)
@@ -109,6 +110,15 @@ class TinyLLamma(nn.Module):
 
 
 def get_model_config(kv_cache_max_len: int = 1024) -> cfg.ModelConfig:
+  """Returns the model config for a TinyLlama model.
+
+  Args:
+    kv_cache_max_len (int): The maximum sequence length of the KV cache. Default
+      is 1024.
+
+  Returns:
+    The model config for a TinyLlama model.
+  """
   attn_config = cfg.AttentionConfig(
       num_heads=32,
       head_dim=64,
@@ -145,7 +155,7 @@ def get_fake_model_config() -> cfg.ModelConfig:
   return config
 
 
-def build_model(checkpoint_path, **kwargs) -> nn.Module:
+def build_model(checkpoint_path: str, **kwargs) -> nn.Module:
   config = get_model_config(**kwargs)
   model = TinyLLamma(config)
   loader = loading_utils.ModelLoader(checkpoint_path, TENSOR_NAMES)
@@ -154,6 +164,8 @@ def build_model(checkpoint_path, **kwargs) -> nn.Module:
 
 
 def define_and_run() -> None:
+  """Instantiates and runs a TinyLlama model."""
+
   current_dir = Path(__file__).parent.resolve()
   tiny_llama_goldens = torch.load(current_dir / "tiny_llama_lm_logits.pt")
   kv_cache_max_len = 1024

ai_edge_torch/generative/test/test_model_conversion.py

@@ -70,35 +70,6 @@ class TestModelConversion(googletest.TestCase):
         )
     )
 
-  @googletest.skipIf(
-      ai_edge_config.Config.use_torch_xla,
-      reason="tests with custom ops are not supported on oss",
-  )
-  def test_toy_model_with_multi_batches(self):
-    self.skipTest("b/362842043")
-    config = toy_model_with_kv_cache.get_model_config()
-    config.batch_size = 2
-    pytorch_model = toy_model_with_kv_cache.ToyModelWithKV(config).eval()
-    idx, input_pos = torch.tensor([[1], [2]], dtype=torch.long), torch.tensor(
-        [10], dtype=torch.int64
-    )
-
-    edge_model = ai_edge_torch.convert(pytorch_model, (idx, input_pos))
-    edge_model.set_interpreter_builder(
-        self._interpreter_builder(edge_model.tflite_model())
-    )
-
-    self.assertTrue(
-        model_coverage.compare_tflite_torch(
-            edge_model,
-            pytorch_model,
-            (idx, input_pos),
-            num_valid_inputs=1,
-            atol=1e-5,
-            rtol=1e-5,
-        )
-    )
-
   @googletest.skipIf(
       ai_edge_config.Config.use_torch_xla,
       reason="tests with custom ops are not supported on oss",

ai_edge_torch/generative/test/test_quantize.py

@@ -25,16 +25,16 @@ from ai_edge_torch.generative.quantize.quant_attrs import Granularity
 from ai_edge_torch.generative.quantize.quant_attrs import Mode
 from ai_edge_torch.quantize import quant_config
 from ai_edge_torch.testing import model_coverage
-from parameterized import parameterized
 import torch
 
 from absl.testing import absltest as googletest
+from absl.testing import parameterized
 
 
-class TestVerifyRecipes(googletest.TestCase):
+class TestVerifyRecipes(parameterized.TestCase):
   """Unit tests that check for model quantization recipes."""
 
-  @parameterized.expand([
+  @parameterized.parameters([
       (Dtype.FP32, Dtype.FP32),
       (Dtype.INT8, Dtype.INT8),
       (Dtype.INT8, Dtype.FP16),
@@ -52,7 +52,7 @@ class TestVerifyRecipes(googletest.TestCase):
           with self.assertRaises(ValueError):
             quant_recipe.LayerQuantRecipe(activation, weight, m, a, g).verify()
 
-  @parameterized.expand([
+  @parameterized.parameters([
       (
           Dtype.FP32,
           Dtype.INT8,
@@ -88,7 +88,7 @@ class TestVerifyRecipes(googletest.TestCase):
     ).verify()
 
 
-class TestQuantizeConvert(googletest.TestCase):
+class TestQuantizeConvert(parameterized.TestCase):
   """Test conversion with quantization."""
 
   def _attention_int8_dynamic_recipe() -> quant_config.QuantConfig:
@@ -105,17 +105,13 @@ class TestQuantizeConvert(googletest.TestCase):
         )
     )
 
-  @parameterized.expand([
+  @parameterized.parameters([
       (quant_recipes.full_fp16_recipe()),
       (quant_recipes.full_int8_dynamic_recipe()),
       (quant_recipes.full_int8_weight_only_recipe()),
       (_attention_int8_dynamic_recipe()),
       (_feedforward_int8_dynamic_recipe()),
   ])
-  @googletest.skipIf(
-      not config.Config.use_torch_xla,
-      reason="Not working with odml_torch at the moment.",
-  )
   def test_quantize_convert_toy_sizes(self, quant_config):
     config = toy_model.get_model_config()
     pytorch_model = toy_model.ToySingleLayerModel(config)
@@ -132,6 +128,23 @@ class TestQuantizeConvert(googletest.TestCase):
         "Quantized model isn't smaller than F32 model.",
     )
 
+  def test_quantize_convert_toy_weight_sharing(self):
+    config = toy_model.get_model_config()
+    pytorch_model = toy_model.ToySingleLayerModelWeightSharing(config)
+    idx = torch.unsqueeze(torch.arange(0, 100), 0)
+    input_pos = torch.arange(0, 100)
+
+    quant_config = quant_recipes.full_int8_dynamic_recipe()
+    quantized_model = ai_edge_torch.convert(
+        pytorch_model, (idx, input_pos), quant_config=quant_config
+    )
+    float_model = ai_edge_torch.convert(pytorch_model, (idx, input_pos))
+    self.assertLess(
+        len(quantized_model._tflite_model),
+        len(float_model._tflite_model),
+        "Quantized model isn't smaller than F32 model.",
+    )
+
   def test_quantize_convert_compare_toy(self):
     self.skipTest("b/338288901")
     config = toy_model_with_kv_cache.get_model_config()

ai_edge_torch/generative/utilities/loader.py

@@ -208,7 +208,7 @@ class ModelLoader:
     if self._file_name.endswith(".safetensors"):
       return load_safetensors
 
-    if self._file_name.endswith(".bin") or self._file_name.endswith(".pt"):
+    if self._file_name.endswith(".bin") or self._file_name.endswith("pt"):
       return load_pytorch_statedict
 
     raise ValueError("File format not supported.")

ai_edge_torch/lowertools/odml_torch_utils.py

@@ -21,6 +21,7 @@ from ai_edge_torch import odml_torch
 from ai_edge_torch._convert import conversion_utils
 from ai_edge_torch._convert import signature as signature_module
 from ai_edge_torch.lowertools import common_utils
+from ai_edge_torch.lowertools import translate_recipe
 from ai_edge_torch.odml_torch import export
 from ai_edge_torch.odml_torch import export_utils
 from ai_edge_torch.quantize import quant_config as qcfg
@@ -186,10 +187,29 @@ def merged_bundle_to_tfl_model(
     converter._experimental_enable_composite_direct_lowering = True
     converter.model_origin_framework = "PYTORCH"
 
+    conversion_utils.set_tfl_converter_quant_flags(converter, quant_config)
+    if (
+        quant_config is not None
+        and quant_config._quantizer_mode
+        == quant_config._QuantizerMode.AI_EDGE_QUANTIZER
+    ):
+      translated_recipe = translate_recipe.translate_to_ai_edge_recipe(
+          quant_config.generative_recipe
+      )
+
     conversion_utils.apply_tfl_converter_flags(converter, _tfl_converter_flags)
 
     tflite_model = converter.convert()
 
+    if (
+        quant_config is not None
+        and quant_config._quantizer_mode
+        == quant_config._QuantizerMode.AI_EDGE_QUANTIZER
+    ):
+      tflite_model = translate_recipe.quantize_model(
+          tflite_model, translated_recipe
+      )
+
   return tflite_model
 
 

ai_edge_torch/lowertools/torch_xla_utils.py

@@ -25,8 +25,8 @@ from typing import Any, Dict, Optional, Tuple, Union
 from ai_edge_torch import model
 from ai_edge_torch._convert import conversion_utils
 from ai_edge_torch._convert import signature as signature_module
-from ai_edge_torch.generative.quantize.ai_edge_quantizer_glue import translate_recipe  # NOQA
 from ai_edge_torch.lowertools import common_utils
+from ai_edge_torch.lowertools import translate_recipe
 from ai_edge_torch.quantize import quant_config as qcfg
 import torch
 from torch_xla import stablehlo

ai_edge_torch/{generative/quantize/ai_edge_quantizer_glue → lowertools}/translate_recipe.py

@@ -17,7 +17,8 @@ from ai_edge_quantizer import quantizer
 from ai_edge_torch.generative.quantize import quant_attrs
 from ai_edge_torch.generative.quantize import quant_recipe
 
-_OpExecutionMode = quantizer.qtyping.OpExecutionMode
+_ComputePrecision = quantizer.qtyping.ComputePrecision
+_QuantGranularity = quantizer.qtyping.QuantGranularity
 _OpName = quantizer.qtyping.TFLOperationName
 _TensorQuantConfig = quantizer.qtyping.TensorQuantizationConfig
 _OpQuantConfig = quantizer.qtyping.OpQuantizationConfig
@@ -50,21 +51,31 @@ def _get_dtype_from_dtype(
     return quantizer.qtyping.TensorDataType.INT
 
 
-def _get_execution_mode_from_mode(mode: quant_attrs.Mode) -> _OpExecutionMode:
+def _get_compute_precision_from_mode(
+    mode: quant_attrs.Mode,
+) -> _ComputePrecision:
   if mode == quant_attrs.Mode.DYNAMIC_RANGE:
-    return _OpExecutionMode.DRQ
+    return _ComputePrecision.INTEGER
   elif mode == quant_attrs.Mode.WEIGHT_ONLY:
-    return _OpExecutionMode.WEIGHT_ONLY
+    return _ComputePrecision.FLOAT
   raise ValueError('Unimplemented execution mode')
 
 
-def _get_channelwise_from_granularity(
+def _get_explicit_dequant_from_mode(mode: quant_attrs.Mode) -> bool:
+  if mode == quant_attrs.Mode.DYNAMIC_RANGE:
+    return False
+  elif mode == quant_attrs.Mode.WEIGHT_ONLY:
+    return True
+  raise ValueError('Unimplemented execution mode')
+
+
+def _get_granularity(
     granularity: quant_attrs.Granularity,
 ) -> bool:
   if granularity == quant_attrs.Granularity.CHANNELWISE:
-    return True
-  elif granularity == quant_attrs.Granularity.NONE:
-    return False
+    return _QuantGranularity.CHANNELWISE
+  if granularity == quant_attrs.Granularity.NONE:
+    return _QuantGranularity.TENSORWISE
   raise ValueError('Unimplemented granularity')
 
 
@@ -88,12 +99,13 @@ def _set_quant_config(
           weight_tensor_config=_TensorQuantConfig(
               num_bits=_get_nbits_from_dtype(layer_recipe.weight_dtype),
               symmetric=True,
-              channel_wise=_get_channelwise_from_granularity(
-                  layer_recipe.granularity
-              ),
+              granularity=_get_granularity(layer_recipe.granularity),
               dtype=_get_dtype_from_dtype(layer_recipe.weight_dtype),
           ),
-          execution_mode=_get_execution_mode_from_mode(layer_recipe.mode),
+          compute_precision=_get_compute_precision_from_mode(layer_recipe.mode),
+          explicit_dequantize=_get_explicit_dequant_from_mode(
+              layer_recipe.mode
+          ),
       ),
       algorithm_key=_get_algorithm_key_from_algorithm(layer_recipe.algorithm),
   )

ai_edge_torch/odml_torch/export.py

@@ -277,7 +277,7 @@ def exported_program_to_mlir(
     main_func.attributes["sym_visibility"] = ir.StringAttr.get("public")
     temp_func.erase()
 
-  module.operation.verify()
+    module.operation.verify()
 
   input_signature = []
   state_dict = {}

ai_edge_torch/version.py

@@ -13,4 +13,4 @@
 # limitations under the License.
 # ==============================================================================
 
-__version__ = "0.3.0.dev20240902"
+__version__ = "0.3.0.dev20240905"

{ai_edge_torch_nightly-0.3.0.dev20240902.dist-info → ai_edge_torch_nightly-0.3.0.dev20240905.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ai-edge-torch-nightly
-Version: 0.3.0.dev20240902
+Version: 0.3.0.dev20240905
 Summary: Supporting PyTorch models with the Google AI Edge TFLite runtime.
 Home-page: https://github.com/google-ai-edge/ai-edge-torch
 Keywords: On-Device ML,AI,Google,TFLite,PyTorch,LLMs,GenAI
@@ -30,7 +30,7 @@ Requires-Dist: tabulate
 Requires-Dist: torch>=2.4.0
 Requires-Dist: torch-xla>=2.4.0
 Requires-Dist: tf-nightly>=2.18.0.dev20240722
-Requires-Dist: ai-edge-quantizer-nightly==0.0.1.dev20240718
+Requires-Dist: ai-edge-quantizer-nightly
 
 Library that supports converting PyTorch models into a .tflite format, which can
 then be run with TensorFlow Lite and MediaPipe.  This enables applications for

{ai_edge_torch_nightly-0.3.0.dev20240902.dist-info → ai_edge_torch_nightly-0.3.0.dev20240905.dist-info}/RECORD

@@ -2,7 +2,7 @@ ai_edge_torch/__init__.py,sha256=48qP37uHT90YPs4eIUQxCiWVwqGEX3idCUs6mQKvX1U,116
 ai_edge_torch/config.py,sha256=PCd9PVrbUNeVIUDFUCnW4goDWU4bjouK28yMYU6VOi0,877
 ai_edge_torch/conftest.py,sha256=r0GTrhMRhlmOGrrkvumHN8hkmyug6WvF60vWq8wRIBI,758
 ai_edge_torch/model.py,sha256=NYV6Mkaje_ditIEI_s_7nLP_-8i4kbGM8nRzieVkbUI,5397
-ai_edge_torch/version.py,sha256=pl_weDdkMIjqukMxBF4uho_z-MvFlGy_ButOq6tJwVc,706
+ai_edge_torch/version.py,sha256=-vQGdl2EaV-VpHRty3RwZzH0UVntVt1tmjhtKOIDscw,706
 ai_edge_torch/_convert/__init__.py,sha256=hHLluseD2R0Hh4W6XZRIXY_dRQeYudjsrKGf6LZz65g,671
 ai_edge_torch/_convert/conversion.py,sha256=kcv_QgNgeyDmrqwdzHicGNP68w6zF7GJg7YkMEIXp4Q,3759
 ai_edge_torch/_convert/conversion_utils.py,sha256=Sr8qXVcTwc-ZnZmK7yxVrIOOp1S_vNrwzC0zUvLTI2o,2160
@@ -42,21 +42,21 @@ ai_edge_torch/generative/examples/__init__.py,sha256=hHLluseD2R0Hh4W6XZRIXY_dRQe
 ai_edge_torch/generative/examples/experimental/__init__.py,sha256=hHLluseD2R0Hh4W6XZRIXY_dRQeYudjsrKGf6LZz65g,671
 ai_edge_torch/generative/examples/experimental/gemma/__init__.py,sha256=hHLluseD2R0Hh4W6XZRIXY_dRQeYudjsrKGf6LZz65g,671
 ai_edge_torch/generative/examples/experimental/gemma/convert_to_tflite.py,sha256=lpiPFSh3SJd6WwuZ0QegSva3__iSz2tUD7L7QfkAe4I,3085
-ai_edge_torch/generative/examples/experimental/gemma/gemma.py,sha256=EdElPCDLYxnNvkPMJkE3WKvESze1ehgShEk2NnbrXLg,7527
+ai_edge_torch/generative/examples/experimental/gemma/gemma.py,sha256=aCoD86pf4nuquUMk7MOR-jsN5FqvySSEuMx9Psxjblk,7261
 ai_edge_torch/generative/examples/experimental/phi/__init__.py,sha256=hHLluseD2R0Hh4W6XZRIXY_dRQeYudjsrKGf6LZz65g,671
 ai_edge_torch/generative/examples/experimental/phi/convert_to_tflite.py,sha256=DavrdGmqUgoThsGNRv3LXMW5tvJdYEvj66Hf1XRqkXU,3055
-ai_edge_torch/generative/examples/experimental/phi/phi2.py,sha256=u-VJX5mjzQKspXtAhNi53LCITtag-3nCaRTKdk5Z1sc,6231
+ai_edge_torch/generative/examples/experimental/phi/phi2.py,sha256=Jxf3ZyYDpS78l6uh4_LGGIcHawrOhZ1vHoHFVxRaK40,6789
 ai_edge_torch/generative/examples/experimental/tiny_llama/__init__.py,sha256=hHLluseD2R0Hh4W6XZRIXY_dRQeYudjsrKGf6LZz65g,671
 ai_edge_torch/generative/examples/experimental/tiny_llama/convert_to_tflite.py,sha256=xPVvHQjLJHFiRv_-Fy2sDm0Aft7SG8SXiV6o3rF03cQ,3108
-ai_edge_torch/generative/examples/experimental/tiny_llama/tiny_llama.py,sha256=zQYtyk3xYdiRAnzMKN58Q_wgTQFnDujxp6L4RFQjiD4,6383
+ai_edge_torch/generative/examples/experimental/tiny_llama/tiny_llama.py,sha256=nUm0SQbCTmNAc5u-C9gbQRFPt7GDvUt6UjH6doTvH-I,6817
 ai_edge_torch/generative/examples/gemma/__init__.py,sha256=hHLluseD2R0Hh4W6XZRIXY_dRQeYudjsrKGf6LZz65g,671
 ai_edge_torch/generative/examples/gemma/convert_gemma2_to_tflite.py,sha256=pseJExH35lSAK0ZtzSHB1sFtRtF_EuT2xcSpGU0gKVI,2524
 ai_edge_torch/generative/examples/gemma/convert_to_tflite.py,sha256=w589IJETATd6Z9_1XCIWbrlCV3E92X_5ac3VVCVFXG0,2522
-ai_edge_torch/generative/examples/gemma/gemma.py,sha256=pzD9dYUYg8E6fFACh-8B8G9NHFXOVEWBjf5aDeipU2s,7202
-ai_edge_torch/generative/examples/gemma/gemma2.py,sha256=ypd6uBb4FgDpuWm_w8JNYBAf4eFxWbYccs8vCgBhi-I,9374
+ai_edge_torch/generative/examples/gemma/gemma.py,sha256=lc1-CfIObHj9D5VJy78BOtGTrQM4TYMI6NfVi8KM5qA,6747
+ai_edge_torch/generative/examples/gemma/gemma2.py,sha256=OcUQLFR136e3QRVXRnmtYnRHXyHJS9EYEFlJ1ymXyRY,8859
 ai_edge_torch/generative/examples/phi2/__init__.py,sha256=hHLluseD2R0Hh4W6XZRIXY_dRQeYudjsrKGf6LZz65g,671
 ai_edge_torch/generative/examples/phi2/convert_to_tflite.py,sha256=ON6zLO-nFS8eJ2yhyWzT5x2Somr-Ca-VjpjT7OGFU10,2506
-ai_edge_torch/generative/examples/phi2/phi2.py,sha256=91mWxEtKgDtUhCAewWNwH_UOOCzy6tPdf6LNRlxZhrc,6700
+ai_edge_torch/generative/examples/phi2/phi2.py,sha256=FFnhv1kx4fHRhSeOreLGj8kAqPnmkz9pD1RRSDVlM_w,6332
 ai_edge_torch/generative/examples/stable_diffusion/__init__.py,sha256=hHLluseD2R0Hh4W6XZRIXY_dRQeYudjsrKGf6LZz65g,671
 ai_edge_torch/generative/examples/stable_diffusion/attention.py,sha256=kDWG6MlIGa89zC5KSRcJlw2c4ITuw8KcchtfmF55f4g,3545
 ai_edge_torch/generative/examples/stable_diffusion/clip.py,sha256=0WniBWQ6_NcQc5WycX3YRRX7Os9AGQSxfc1m2HKBqg8,4479
@@ -77,12 +77,12 @@ ai_edge_torch/generative/examples/t5/convert_to_tflite.py,sha256=CZVuNEL8OHPkdsz
 ai_edge_torch/generative/examples/t5/t5.py,sha256=Zobw5BV-PC0nlU9Z6fzb2O07rMeU8vGIk-KtKp9D_H0,20871
 ai_edge_torch/generative/examples/t5/t5_attention.py,sha256=1lvbSlzyBwmd5Bs7-Up_v4iJQkCPIJx2RmMkLgy7l2Q,8508
 ai_edge_torch/generative/examples/test_models/__init__.py,sha256=hHLluseD2R0Hh4W6XZRIXY_dRQeYudjsrKGf6LZz65g,671
-ai_edge_torch/generative/examples/test_models/toy_model.py,sha256=LfWO_gSr1f66V1pxAc6yh21mtaJs7TVeuO9748zXBnE,3963
+ai_edge_torch/generative/examples/test_models/toy_model.py,sha256=5wj2RmQRIwD6O_R_pp-A_7gKGSdHWDSXyis97r1ELVI,5622
 ai_edge_torch/generative/examples/test_models/toy_model_with_external_kv_cache.py,sha256=l9swUKTcDtnTibNSNExaMgLvDeJ4Er2tVh5ZW1EtRgk,5809
 ai_edge_torch/generative/examples/test_models/toy_model_with_kv_cache.py,sha256=mQkcpSe6HlRLMkIRCEHc9ZXL7jxEp9RWSGUQjjd-r2w,4841
 ai_edge_torch/generative/examples/tiny_llama/__init__.py,sha256=hHLluseD2R0Hh4W6XZRIXY_dRQeYudjsrKGf6LZz65g,671
 ai_edge_torch/generative/examples/tiny_llama/convert_to_tflite.py,sha256=CLRqO7ycMbpy7J3_Czp1sLx6hcdwGD9zVq04yRba0e8,2550
-ai_edge_torch/generative/examples/tiny_llama/tiny_llama.py,sha256=JmwU1sniO37vnCFc8dklbd-0ofTZK0PaBv_Ksn1Vq6M,5930
+ai_edge_torch/generative/examples/tiny_llama/tiny_llama.py,sha256=4ku0ni3MOWamhPrzLap0BmtdNFk7CH0hwjPNoRAKpvQ,6278
 ai_edge_torch/generative/fx_passes/__init__.py,sha256=fmNNXawJ722M4cTUuTx289rT0NHxBEsOy_k8baqCOms,1173
 ai_edge_torch/generative/fx_passes/remove_sdpa_zero_mask_pass.py,sha256=sXis0U4u-RoIp_NyrmWJNnqFqpqRuZOrhfsJIO6rMps,2028
 ai_edge_torch/generative/layers/__init__.py,sha256=hHLluseD2R0Hh4W6XZRIXY_dRQeYudjsrKGf6LZz65g,671
@@ -106,16 +106,14 @@ ai_edge_torch/generative/quantize/quant_recipe.py,sha256=tKnuJq6hPD23JPCB9nPAlE1
 ai_edge_torch/generative/quantize/quant_recipe_utils.py,sha256=4fgmP_GgeiFUOkIaC9ZZXC12eO3DQZdrWDXRz5YXiwU,2270
 ai_edge_torch/generative/quantize/quant_recipes.py,sha256=0Kvr_o7pbMnE8VMe6Ml0FBxkHM6RJ3C14B2I1mjItjc,2030
 ai_edge_torch/generative/quantize/supported_schemes.py,sha256=FjdycEOvxRgBmQdZVufetPvkDoD7rUowIOSKV9oV5Kk,1418
-ai_edge_torch/generative/quantize/ai_edge_quantizer_glue/__init__.py,sha256=hHLluseD2R0Hh4W6XZRIXY_dRQeYudjsrKGf6LZz65g,671
-ai_edge_torch/generative/quantize/ai_edge_quantizer_glue/translate_recipe.py,sha256=sSHc_4hUEvi-3KmqbpqWbrRKBjCI1AOctM3dr2EH3vk,5263
 ai_edge_torch/generative/test/__init__.py,sha256=hHLluseD2R0Hh4W6XZRIXY_dRQeYudjsrKGf6LZz65g,671
 ai_edge_torch/generative/test/test_experimental_ekv.py,sha256=8qv_eVtJW9GPvBEf2hPQe3tpdJ33XShya6MCX1FqrZM,4355
 ai_edge_torch/generative/test/test_loader.py,sha256=_y5EHGgoNOmCuYonsB81UJScHVsTAQXUVd44czMAw6k,3379
-ai_edge_torch/generative/test/test_model_conversion.py,sha256=wQLVjMnKHBCVCU_I-xAUZvlOFoDiwYwKQDvCZ2mjtOM,6193
+ai_edge_torch/generative/test/test_model_conversion.py,sha256=KZ0uCeOdKMKyW8jBE8aOjweZmws4mvz37u8zH4XayVU,5285
 ai_edge_torch/generative/test/test_model_conversion_large.py,sha256=o3l7HFHP-sg8aHeLNTSpMF91YovPODjp4QzYUnSJiIE,4479
-ai_edge_torch/generative/test/test_quantize.py,sha256=JEsk9SAkHK0SFm44K_quISc5yBBS6yvtBP1MDyFHdFw,5344
+ai_edge_torch/generative/test/test_quantize.py,sha256=kY_NRpF-v1i4clqI1CFFWEagJv-5PzBDkeJ2fInl9_w,5913
 ai_edge_torch/generative/utilities/__init__.py,sha256=-_jxnnFnCgnTU4oTm4MnRsvL5lqhomBNdFBbqfmfHPo,720
-ai_edge_torch/generative/utilities/loader.py,sha256=QFZ2lkeoYQ9MZ1CAFVxBHG4OT192SH74UtJCvbDsdeI,12727
+ai_edge_torch/generative/utilities/loader.py,sha256=6J0aAP6-6LySeqeYIHKcchr5T9cVtSO34aoDr3V9gxY,12726
 ai_edge_torch/generative/utilities/stable_diffusion_loader.py,sha256=pKp3AMSbS3otCvgwJRF5M1l4JRNKk-aCKimXzIMSrds,35679
 ai_edge_torch/generative/utilities/t5_loader.py,sha256=_UXcc1QKT-S92hikfo-fTBFhnYLzROqcyRqKonVsqj4,16885
 ai_edge_torch/hlfb/__init__.py,sha256=sH4um75na-O8tzxN6chFyp6Y4xnexsE7kUQpZySv6dE,735
@@ -128,13 +126,14 @@ ai_edge_torch/hlfb/test/test_stablehlo_composite_builder.py,sha256=j8WpeS-mz3Zr4
 ai_edge_torch/lowertools/__init__.py,sha256=A8WBXvWtuFYYWtNTqPD7waVntLaSVAnSMwx5ugjZBIw,761
 ai_edge_torch/lowertools/_shim.py,sha256=ilL7x1ebUBj1clg7bagrX4y_nVSHiGrvDrOVfuTeenE,3039
 ai_edge_torch/lowertools/common_utils.py,sha256=Z7p-ivOHtddktpnHrlDm_dSoTxJOdEjFXIGQbzjgwQo,4504
-ai_edge_torch/lowertools/odml_torch_utils.py,sha256=GKfW1X-QSFffQdVlBuD-bNpP265xcdUlfBY3-9I4f_o,7447
+ai_edge_torch/lowertools/odml_torch_utils.py,sha256=K5dZ_fFDL3GWKo0IoY4OC_GX5MY-guY-MqteolyV9hg,8098
 ai_edge_torch/lowertools/test_utils.py,sha256=bPgc2iXX16KYtMNvmsRdKfrCY6UJmcfitfCOvHoD7Oc,1930
-ai_edge_torch/lowertools/torch_xla_utils.py,sha256=-SRm9YNsIGsaVd5Cyp2PP-tdLBJH8EDoMFAa2y89a1w,9043
+ai_edge_torch/lowertools/torch_xla_utils.py,sha256=n6G3pFGmHar7kgKDsdTB74kv1PUuTTu1XjV7R-QizzE,9003
+ai_edge_torch/lowertools/translate_recipe.py,sha256=DNzD0VD35YZDqiZjAF1IyIPSzUGPDpE0jvFCCYIzpnc,5667
 ai_edge_torch/odml_torch/__init__.py,sha256=S8jOzE9nLof-6es3XDiGJRN-9H_XTxsVm9dE7lD3RWo,812
 ai_edge_torch/odml_torch/_torch_future.py,sha256=jSYHf1CMTJzMizPMbu2b39hAt0ZTR6gQLq67GMe9KTo,2336
 ai_edge_torch/odml_torch/_torch_library.py,sha256=Lw1gqL2HWNRspdTwNhIkYAHDyafHedHtkXyKKxn-Wss,805
-ai_edge_torch/odml_torch/export.py,sha256=hIGT-JKYbIa6e_G0AD-k4MSTIAMGdHC1hNHMn9CxsYw,10467
+ai_edge_torch/odml_torch/export.py,sha256=OXN6jipwFtBvQ9XdyeDGQTQ_-UnCxPYnLc_WW7xF0aI,10469
 ai_edge_torch/odml_torch/export_utils.py,sha256=q84U69ZQ82hLXw-xncJ8IW-K71Xux-NWlzZTs7hdZWA,5127
 ai_edge_torch/odml_torch/tf_integration.py,sha256=lTFJPPEijLPFmn6qq2jbpVTQOo0YaOTK36kK6rCiyIE,5956
 ai_edge_torch/odml_torch/composite/__init__.py,sha256=71GM_gDZxJyo38ZSoYSwhZX3xKA9rknO93JS9kw9w_c,778
@@ -162,8 +161,8 @@ ai_edge_torch/quantize/quant_config.py,sha256=U0KisSW-uZkoMJcy-ZP9W57p3tsa594fr9
 ai_edge_torch/testing/__init__.py,sha256=hHLluseD2R0Hh4W6XZRIXY_dRQeYudjsrKGf6LZz65g,671
 ai_edge_torch/testing/model_coverage/__init__.py,sha256=5P8J6Zk5YYtDvTBucFvB9NGSRI7Gw_24WnrbhXgycEE,765
 ai_edge_torch/testing/model_coverage/model_coverage.py,sha256=UPB448aMDUyC0HNYVqio2rcJPnDN0tBQMP08J6vPYew,4718
-ai_edge_torch_nightly-0.3.0.dev20240902.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-ai_edge_torch_nightly-0.3.0.dev20240902.dist-info/METADATA,sha256=Bvc6_uRgjiaqUsVareqJETErsc5rU7NNTPTDqn0JwoA,1878
-ai_edge_torch_nightly-0.3.0.dev20240902.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-ai_edge_torch_nightly-0.3.0.dev20240902.dist-info/top_level.txt,sha256=5KXRaF2hwkApYxf7Y8y_tVb9aulGTlbOoNdbx1aKRkE,14
-ai_edge_torch_nightly-0.3.0.dev20240902.dist-info/RECORD,,
+ai_edge_torch_nightly-0.3.0.dev20240905.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+ai_edge_torch_nightly-0.3.0.dev20240905.dist-info/METADATA,sha256=8yrrm7TEYgaRhKdUwgStjCqrTWs8YcnnlzoTJt2NrJg,1859
+ai_edge_torch_nightly-0.3.0.dev20240905.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+ai_edge_torch_nightly-0.3.0.dev20240905.dist-info/top_level.txt,sha256=5KXRaF2hwkApYxf7Y8y_tVb9aulGTlbOoNdbx1aKRkE,14
+ai_edge_torch_nightly-0.3.0.dev20240905.dist-info/RECORD,,

ai_edge_torch/generative/quantize/ai_edge_quantizer_glue/__init__.py

@@ -1,14 +0,0 @@
-# Copyright 2024 The AI Edge Torch Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================