mediapipe-nightly 0.10.11.post20240302__cp39-cp39-macosx_11_0_universal2.whl → 0.10.11.post20240303__cp39-cp39-macosx_11_0_universal2.whl

mediapipe/__init__.py

@@ -23,4 +23,4 @@ del modules
 del python
 del mediapipe
 del util
-__version__ = '0.10.11-20240302'
+__version__ = '0.10.11-20240303'

mediapipe/tasks/python/genai/converter/llm_converter.py

@@ -1,5 +1,6 @@
 """Functions to perform the checkpoint conversion."""
 
+import contextlib
 import os
 from typing import List, Optional
 
@@ -168,16 +169,41 @@ def convert_bpe_vocab(vocab_model_file: str, output_dir: str) -> str:
   return output_vocab_file
 
 
+@contextlib.contextmanager
+def filemanager(filename: str, mode: str):
+  try:
+    with open(filename, mode) as f:
+      yield f
+  finally:
+    pass
+
+
+def sort_layer_info(layer_info_file: str) -> None:
+  """Loads and sorts the layer info file."""
+  layer_info = []
+  with filemanager(layer_info_file, 'r') as finfo:
+    for line in finfo:
+      line = line.strip()
+      if line:
+        layer_info.append(line)
+  layer_info = list(set(layer_info))
+  layer_info.sort()
+  with filemanager(layer_info_file, 'w') as finfo:
+    for line in layer_info:
+      finfo.write(line + '\n')
+      finfo.write('\n')
+
+
 def convert_checkpoint(config: ConversionConfig) -> None:
   """Converts the checkpoint to tflite file."""
   logging.info('input folder: %s', config.input_ckpt)
 
-  if config.model_type == 'GEMMA_2B':
-    vocab_model_path = config.vocab_model_file
-  else:
+  if os.path.isdir(config.vocab_model_file):
     vocab_model_path = convert_bpe_vocab(
         config.vocab_model_file, config.output_dir
     )
+  else:
+    vocab_model_path = config.vocab_model_file
 
   if not config.combine_file_only:
     # Load the layer weights and prepare the quantization configurations.
@@ -209,6 +235,8 @@ def convert_checkpoint(config: ConversionConfig) -> None:
       del quantized_tensors
       del writer
 
+    sort_layer_info(os.path.join(config.output_dir, 'layer_info.txt'))
+
   combined_weight_bins_to_tflite(
       config.model_type,
       config.backend,

mediapipe/tasks/python/genai/converter/safetensors_converter.py

@@ -393,6 +393,14 @@ class GemmaMapper(converter_base.LayerActionMapperBase):
           quantize_axis = [1]
       elif layer_type == LayerType.EMBEDDING:
         quantize_bits = self._embedding_quant_bits
+        if tensor_value.shape[1] < 256128:
+          # Padd the embedding shape if the vocab size was smaller than the
+          # original setting: 256128.
+          pad_count = 256128 - tensor_value.shape[1]
+          tensor_value = np.pad(
+              tensor_value, ((0, 0), (0, pad_count)), mode="constant"
+          )
+
     target_name = self.update_target_name(layer_name)
 
     actions = [

mediapipe/tasks/python/genai/converter/weight_bins_writer.py

@@ -106,7 +106,7 @@ class WeightBinsWriter(converter_base.ModelWriterBase):
       # Sort weights_info
       weights_info.sort()
       with filemanager(
-          os.path.join(self._output_dir, 'layer_info.txt'), 'w'
+          os.path.join(self._output_dir, 'layer_info.txt'), 'a'
       ) as finfo:
         for line in weights_info:
           finfo.write(line + '\n')

mediapipe/version.txt

@@ -1 +1 @@
-0.10.11-20240302
+0.10.11-20240303

{mediapipe_nightly-0.10.11.post20240302.dist-info → mediapipe_nightly-0.10.11.post20240303.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mediapipe-nightly
-Version: 0.10.11.post20240302
+Version: 0.10.11.post20240303
 Summary: MediaPipe is the simplest way for researchers and developers to build world-class ML solutions and applications for mobile, edge, cloud and the web.
 Home-page: https://github.com/google/mediapipe
 Author: The MediaPipe Authors

{mediapipe_nightly-0.10.11.post20240302.dist-info → mediapipe_nightly-0.10.11.post20240303.dist-info}/RECORD

@@ -1,9 +1,4 @@
-mediapipe_nightly-0.10.11.post20240302.dist-info/RECORD,,
-mediapipe_nightly-0.10.11.post20240302.dist-info/LICENSE,sha256=hwfu8FM5h-_FsVXWR2HutuIHk_ULm9Gmja0c9HGdDtg,12331
-mediapipe_nightly-0.10.11.post20240302.dist-info/WHEEL,sha256=Roq3jd81zxBo_pMxKZzXrRts-yky38mkvNnjvr99fPU,109
-mediapipe_nightly-0.10.11.post20240302.dist-info/top_level.txt,sha256=LG-epD1oIiiHFRqLp--7jacjB3dbx2RfMcLYjCIhmxU,175
-mediapipe_nightly-0.10.11.post20240302.dist-info/METADATA,sha256=Z_ixYoKtEPje2xOmQuHu44KTR17vwvA_rhw1LA27h1E,9715
-mediapipe/__init__.py,sha256=FG3p8a-sPsr3_nru1WF1PkqKrAHldPzVed4uOfnptR8,816
+mediapipe/__init__.py,sha256=EXN6Y5HrB1DAy9ze__JS6IX9nIpv7DrRL1sPkN6BX6Y,816
 mediapipe/tasks/__init__.py,sha256=sVJS2p8J2PNVl8DLRPVY6KLpHenP_z3QVPRU0x_iL5g,571
 mediapipe/tasks/python/__init__.py,sha256=wIM_WOWboOVI1MeehN8fkN_DjoA0MEBVw5mShAd8AS4,858
 mediapipe/tasks/python/benchmark/__init__.py,sha256=epEucluzX0HinwBZoS7Tgb19j_qgfTuBf-vBkqemch8,587
@@ -89,15 +84,15 @@ mediapipe/tasks/python/genai/__init__.py,sha256=7rri6fT6wNurla8O2c5yKiLs9_3qIY0v
 mediapipe/tasks/python/genai/converter/pytorch_converter_test.py,sha256=y_Mg9pOQtlUDh6uVmkz5LcUbk-pmDLA9L3KcxKR-OaA,3041
 mediapipe/tasks/python/genai/converter/safetensors_converter_test.py,sha256=oCk4FnsjBJkEPlXtv8fdq9dn3I06LsSQRMi0BV_9mew,2802
 mediapipe/tasks/python/genai/converter/quantization_util.py,sha256=B6i13GqRRIwMabEJWO8rFHPMBjIgdOhFpHiwMD4GzRc,17196
-mediapipe/tasks/python/genai/converter/safetensors_converter.py,sha256=ttCpqyMeHkIFzppP8G8uMG5VsslK5-ULFT5U3R_qif0,18713
+mediapipe/tasks/python/genai/converter/safetensors_converter.py,sha256=C7C2ZpZmryDwCZyLNzaAKv-n8klctlLB8jZ2-44_BHY,19039
 mediapipe/tasks/python/genai/converter/weight_bins_writer_test.py,sha256=6qgNYXODNOsbveZ0ighEW4JBdawil9mPcC16MZ0mdm8,1994
 mediapipe/tasks/python/genai/converter/converter_base.py,sha256=1nBgvcY5xaI0ZPjBJTVdViDFraVyJUZHLXBT0dYOX9c,6568
 mediapipe/tasks/python/genai/converter/pytorch_converter.py,sha256=b-GWYOzgD-ZRGgyqcXE9LG_JOL0Mqba4q0pc_imrbrg,10771
 mediapipe/tasks/python/genai/converter/__init__.py,sha256=jfUkinDJR5BVldnbJMbo5vIr2Xc5Z4TTnaCJTNoAUvg,893
 mediapipe/tasks/python/genai/converter/converter_factory.py,sha256=2K16PZBQym0WhXM2HOdBMHMugykohoD4OTaOIo-UKko,2928
-mediapipe/tasks/python/genai/converter/llm_converter.py,sha256=OJqrYEDPQYwoOftBw-Ajsqtkcj9uRHBAb_Oo_RCEKa4,7896
+mediapipe/tasks/python/genai/converter/llm_converter.py,sha256=ZaPNzH94zTtXOL6Zrei6Wq1lMZeR9MF-1xuGP_abj28,8596
 mediapipe/tasks/python/genai/converter/quantization_util_test.py,sha256=ICujhTFeREGuHGmNk1PlBpf1AUThFvv-Wl5UuZ-xWAk,9060
-mediapipe/tasks/python/genai/converter/weight_bins_writer.py,sha256=qrsjTWw99u-VDNhtHbnbDfMypx4sGfGyY8mMBXYnQtA,4347
+mediapipe/tasks/python/genai/converter/weight_bins_writer.py,sha256=4nUVWlvR6CMFCgSTb5ckdiFiLQpF4rrCfZTuZCTwkaQ,4347
 mediapipe/tasks/python/metadata/metadata.py,sha256=EECQnM-Af0angD60jaBBOuNMgt7HExH6SqVtVMFNHGc,33763
 mediapipe/tasks/python/metadata/metadata_displayer_cli.py,sha256=tLhF0B1mXG0igFTA9nPh8t1efRpRw2hQ00XpTPYdk_o,1202
 mediapipe/tasks/python/metadata/__init__.py,sha256=YGHXQMz1ZGPcNgSXggu03b0USZKE8d9Xqvn6NDUl898,586
@@ -320,7 +315,7 @@ mediapipe/util/analytics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZ
 mediapipe/util/analytics/mediapipe_log_extension_pb2.py,sha256=UX1x8jckc8NQbUz-SXOKkb8co3JNIRX3rfaMs4_Dz8g,4319
 mediapipe/util/analytics/mediapipe_logging_enums_pb2.py,sha256=9pxs-DNSQnXQe7E_LKokGDbD_G1FpmzzJw1jzex78lU,3781
 mediapipe/python/solution_base.py,sha256=nEIqsho9DlutfvWWzdSxCOpJ2QzN7n2938WLDmFzn38,26072
-mediapipe/python/_framework_bindings.cpython-39-darwin.so,sha256=1QHgiXRi2935VpdMBmqvg_ADecPdZ4e72B83LPl_Oos,78906451
+mediapipe/python/_framework_bindings.cpython-39-darwin.so,sha256=w4h6YecKQD3ke494B6ka0Lcjydy8e8Wqu8nCTTlbL7M,78906451
 mediapipe/python/timestamp_test.py,sha256=oWKTZMsV586jH57OBV30rihcymETyGC29VbYURNLJQQ,2528
 mediapipe/python/image_frame_test.py,sha256=ZSjdE-an2t8i6MiA4_Xri91VMH5_CCx45fjhWUQptMY,8602
 mediapipe/python/__init__.py,sha256=BQglgytZUe7_ZuD8amosz-szWdJ2LQp81nsuiEY3W84,1493
@@ -578,3 +573,8 @@ mediapipe/modules/selfie_segmentation/selfie_segmentation.tflite,sha256=nuFo7HyP
 mediapipe/modules/palm_detection/palm_detection_full.tflite,sha256=GxTpQixq0AbN5lgaRsi5DdVzwHq385NLVYnnzqP4mlQ,2339846
 mediapipe/modules/palm_detection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 mediapipe/modules/palm_detection/palm_detection_lite.tflite,sha256=6aSq3fkN2laocjUwPPAOTC0_socl9o_Yh3KZfayQXBg,1985440
+mediapipe_nightly-0.10.11.post20240303.dist-info/RECORD,,
+mediapipe_nightly-0.10.11.post20240303.dist-info/LICENSE,sha256=hwfu8FM5h-_FsVXWR2HutuIHk_ULm9Gmja0c9HGdDtg,12331
+mediapipe_nightly-0.10.11.post20240303.dist-info/WHEEL,sha256=Roq3jd81zxBo_pMxKZzXrRts-yky38mkvNnjvr99fPU,109
+mediapipe_nightly-0.10.11.post20240303.dist-info/top_level.txt,sha256=LG-epD1oIiiHFRqLp--7jacjB3dbx2RfMcLYjCIhmxU,175
+mediapipe_nightly-0.10.11.post20240303.dist-info/METADATA,sha256=PzIBKWQFkkKcurgEWrf-xP3DXRNd6bRRmRsDXPMMvaQ,9715