bigdl-core-cpp 2.5.0b20240818__py3-none-manylinux2010_x86_64.whl → 2.5.0b20240820__py3-none-manylinux2010_x86_64.whl

bigdl/cpp/convert-hf-to-gguf.py

@@ -1570,6 +1570,33 @@ class LlamaModel(Model):
         return [(self.map_tensor_name(name), data_torch)]
 
     def prepare_tensors(self):
+        if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):
+            if rope_scaling.get("rope_type", '').lower() == "llama3":
+                base = self.hparams.get("rope_theta", 10000.0)
+                dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
+                freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
+                factor = rope_scaling.get("factor", 8.0)
+                low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)
+                high_freq_factor = rope_scaling.get("high_freq_factor", 4.0)
+                old_context_len = self.hparams.get("original_max_position_embeddings", 8192)
+
+                low_freq_wavelen = old_context_len / low_freq_factor
+                high_freq_wavelen = old_context_len / high_freq_factor
+                assert low_freq_wavelen != high_freq_wavelen
+
+                rope_factors = []
+                for freq in freqs:
+                    wavelen = 2 * math.pi / freq
+                    if wavelen < high_freq_wavelen:
+                        rope_factors.append(1)
+                    elif wavelen > low_freq_wavelen:
+                        rope_factors.append(factor)
+                    else:
+                        smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)
+                        rope_factors.append(1 / ((1 - smooth) / factor + smooth))
+
+                self.gguf_writer.add_tensor(self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), np.array(rope_factors, dtype=np.float32))
+
         super().prepare_tensors()
 
         if self._experts is not None:
@@ -3686,4 +3713,4 @@ def main() -> None:
 
 
 if __name__ == '__main__':
-    main()
+    main()

{bigdl_core_cpp-2.5.0b20240818.dist-info → bigdl_core_cpp-2.5.0b20240820.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bigdl-core-cpp
-Version: 2.5.0b20240818
+Version: 2.5.0b20240820
 Summary: Large Language Model Develop Toolkit
 Author: BigDL Authors
 License: Apache License, Version 2.0

bigdl_core_cpp-2.5.0b20240820.dist-info/RECORD

@@ -0,0 +1,45 @@
+bigdl/cpp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bigdl/cpp/convert-hf-to-gguf.py,sha256=ci596u_GY7MDZrPeUGJ3TnYYzdZ2_1e0JT-nLeNpps8,168262
+bigdl/cpp/convert.py,sha256=XMMcpfWHwEAAWzwLXe9mmJTU7cMvcyw8g2BFctfZnvI,69417
+bigdl/cpp/cli/init-llama-cpp,sha256=qpIwMQc2Vb11fmovXFJKefp5iD1tQMfL8Ma7Y3Ebguk,400
+bigdl/cpp/cli/init-ollama,sha256=5eg4DsWg87iysFRh7leawr6H47F_sHydwL5we6fUG6o,193
+bigdl/cpp/gguf-py/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bigdl/cpp/gguf-py/gguf/__init__.py,sha256=LUfJeRbrLY6TgwI0rLw_qo8klu7GgYB8Ycu8H4V0eb8,218
+bigdl/cpp/gguf-py/gguf/constants.py,sha256=WRloarpYU4sc0tIi8aDgIh6xb6Ox0CPLCiPa9Yzzm2Q,46902
+bigdl/cpp/gguf-py/gguf/gguf.py,sha256=MzR6GNwyrWBN3w-o995FY0qX8U3J9Mbinu9HAMEjx8g,477
+bigdl/cpp/gguf-py/gguf/gguf_reader.py,sha256=5yv15RSKLfz9ViQd2e-vQxzXEdCGqgJfGfoBbzPYTHw,12364
+bigdl/cpp/gguf-py/gguf/gguf_writer.py,sha256=HqxGAplbhGDgn9bELLFFh3Nf15Gg16BOTVvhOMKe6GM,34612
+bigdl/cpp/gguf-py/gguf/lazy.py,sha256=yMcMTIrlwpQrX9m8YSRdvyHsNupHZMSj1RsjAAPrxqE,8527
+bigdl/cpp/gguf-py/gguf/metadata.py,sha256=HNu6fzuLLSzQ_K_xJkKqtT5Tni3kGESj67BZ2uPNajg,24851
+bigdl/cpp/gguf-py/gguf/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bigdl/cpp/gguf-py/gguf/quants.py,sha256=dI6JsSpaSzfdz7OwfqH1kvK98M_Ax8d6NMNYY5pyoQM,4354
+bigdl/cpp/gguf-py/gguf/tensor_mapping.py,sha256=1tNoM-XXvD8JaStxt39HBRe_obCWawHfTFVCWXRe9Zk,30147
+bigdl/cpp/gguf-py/gguf/utility.py,sha256=xEIfT3CphqAyI56A5XZQDMti2XGG5k6kxm5bjf_Vn-E,2933
+bigdl/cpp/gguf-py/gguf/vocab.py,sha256=vsRrPQ6RafirhBmMp9h3YGZh_tqEjeg7S2hiLvMv7Ys,6847
+bigdl/cpp/libs/baby-llama,sha256=u5ibSMXHCYc2uIqsPtTFL_JKOFmTBXRxY2yCWkbp4Ic,14319136
+bigdl/cpp/libs/batched,sha256=OugBdAdLzSLhbzMnltk3xZqsDM8xnkQU5dvix3pBPcA,14243704
+bigdl/cpp/libs/batched-bench,sha256=Z0z-USL1rvw1iKDGMnFdetYCTmZvk-T_8g386b_bpLA,14243864
+bigdl/cpp/libs/benchmark,sha256=GEtYJQCxe8aniNOzxFP8RXdENg5VRkJvpFJ8s_jhoJU,1452712
+bigdl/cpp/libs/embedding,sha256=VBP6-wea7RRNMgIoferFs08aKiyjbBV31h95EswkePk,14239752
+bigdl/cpp/libs/gguf,sha256=TfHkBvibkVzp6IVjHeyINMr9FJkDMbx-kc_jHRSOoLI,12483008
+bigdl/cpp/libs/imatrix,sha256=Werf4flSXxriskdr-rZp95PINBsC_8OUumacIDHSMhs,14267608
+bigdl/cpp/libs/llama-bench,sha256=6rCmJO7ddWjdOu59LAGTzmdAMePgAleU9V6LncV0o-E,14333248
+bigdl/cpp/libs/llava-cli,sha256=g5TIcfjV0tX7OwMJm1bTEQQVmqKyv872m4t-SHmojiY,14557832
+bigdl/cpp/libs/lookahead,sha256=0pRIwO9QLbDZiryzljR6AwvHs320wS6n6_Z5NvfphSE,14252072
+bigdl/cpp/libs/lookup,sha256=2nwJhdR0ej86-m1tcWIm1aJL8wuVx55IlidEzg5xEBQ,14271856
+bigdl/cpp/libs/ls-sycl-device,sha256=o5dVh3SI1dEgaU_L9euiEMqmkt51RNQ6nk2inKSOJmI,12478752
+bigdl/cpp/libs/main,sha256=oQJPS60tjFuxdY_hrseOpVGO_tnWhJFrTqyVSn4cLlE,14299192
+bigdl/cpp/libs/ollama,sha256=7hw1hFmbRCLDJReMFybHE5OXoqRHwqpJgflKFl3-RH8,47374520
+bigdl/cpp/libs/perplexity,sha256=Tb6AP9w1K-bBmZDQl7jTo1p8HbBTElMWv6oWasWqlxs,14340696
+bigdl/cpp/libs/quantize,sha256=Vuk-3RYNvVt7FKqhx8OVtErQEH2wOLhXJlMbBpkabkI,14265984
+bigdl/cpp/libs/quantize-stats,sha256=P5bGwOKJlLD2fWHU6VM5F9xMDuGVaNvja3JZAIXYRqk,13780224
+bigdl/cpp/libs/save-load-state,sha256=zBR2iKN28tQyd038Qj-NotfAn0BZAWhQqgjfgNa8UK4,14239880
+bigdl/cpp/libs/server,sha256=4d5gERJu7qIrb8kONTKLGAnLeN9NptL4MUjdVQpzlJo,15097272
+bigdl/cpp/libs/speculative,sha256=2OZezaJeae0__HTXf6oy8ep3hFycpImRfxgYOneYS9g,14278112
+bigdl/cpp/libs/tokenize,sha256=KDYXYerowPEau6QXkTy8gOFs7O21Cnbvjs6s8-f1n4c,14239800
+bigdl_core_cpp-2.5.0b20240820.data/scripts/init-llama-cpp,sha256=qpIwMQc2Vb11fmovXFJKefp5iD1tQMfL8Ma7Y3Ebguk,400
+bigdl_core_cpp-2.5.0b20240820.data/scripts/init-ollama,sha256=5eg4DsWg87iysFRh7leawr6H47F_sHydwL5we6fUG6o,193
+bigdl_core_cpp-2.5.0b20240820.dist-info/METADATA,sha256=bSz40OvUMWpyVy-RJbbzAjIv7qZm0VnDNaxx5eSZ8Eg,643
+bigdl_core_cpp-2.5.0b20240820.dist-info/WHEEL,sha256=LP9LuSkp9EkkS3GaxGDqD2GVQk9RkMkUIxzcmUzigkU,108
+bigdl_core_cpp-2.5.0b20240820.dist-info/top_level.txt,sha256=iGuLfZARD_qANcIMfy0tbbrC3EtCg6BSiH8icc3dLWs,6
+bigdl_core_cpp-2.5.0b20240820.dist-info/RECORD,,

bigdl_core_cpp-2.5.0b20240818.dist-info/RECORD

@@ -1,45 +0,0 @@
-bigdl/cpp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bigdl/cpp/convert-hf-to-gguf.py,sha256=1wAaTiqtAsnA_4k10TCzDSp4KTZ_mldv6pY47z5gg7M,166672
-bigdl/cpp/convert.py,sha256=XMMcpfWHwEAAWzwLXe9mmJTU7cMvcyw8g2BFctfZnvI,69417
-bigdl/cpp/cli/init-llama-cpp,sha256=qpIwMQc2Vb11fmovXFJKefp5iD1tQMfL8Ma7Y3Ebguk,400
-bigdl/cpp/cli/init-ollama,sha256=5eg4DsWg87iysFRh7leawr6H47F_sHydwL5we6fUG6o,193
-bigdl/cpp/gguf-py/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bigdl/cpp/gguf-py/gguf/__init__.py,sha256=LUfJeRbrLY6TgwI0rLw_qo8klu7GgYB8Ycu8H4V0eb8,218
-bigdl/cpp/gguf-py/gguf/constants.py,sha256=WRloarpYU4sc0tIi8aDgIh6xb6Ox0CPLCiPa9Yzzm2Q,46902
-bigdl/cpp/gguf-py/gguf/gguf.py,sha256=MzR6GNwyrWBN3w-o995FY0qX8U3J9Mbinu9HAMEjx8g,477
-bigdl/cpp/gguf-py/gguf/gguf_reader.py,sha256=5yv15RSKLfz9ViQd2e-vQxzXEdCGqgJfGfoBbzPYTHw,12364
-bigdl/cpp/gguf-py/gguf/gguf_writer.py,sha256=HqxGAplbhGDgn9bELLFFh3Nf15Gg16BOTVvhOMKe6GM,34612
-bigdl/cpp/gguf-py/gguf/lazy.py,sha256=yMcMTIrlwpQrX9m8YSRdvyHsNupHZMSj1RsjAAPrxqE,8527
-bigdl/cpp/gguf-py/gguf/metadata.py,sha256=HNu6fzuLLSzQ_K_xJkKqtT5Tni3kGESj67BZ2uPNajg,24851
-bigdl/cpp/gguf-py/gguf/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bigdl/cpp/gguf-py/gguf/quants.py,sha256=dI6JsSpaSzfdz7OwfqH1kvK98M_Ax8d6NMNYY5pyoQM,4354
-bigdl/cpp/gguf-py/gguf/tensor_mapping.py,sha256=1tNoM-XXvD8JaStxt39HBRe_obCWawHfTFVCWXRe9Zk,30147
-bigdl/cpp/gguf-py/gguf/utility.py,sha256=xEIfT3CphqAyI56A5XZQDMti2XGG5k6kxm5bjf_Vn-E,2933
-bigdl/cpp/gguf-py/gguf/vocab.py,sha256=vsRrPQ6RafirhBmMp9h3YGZh_tqEjeg7S2hiLvMv7Ys,6847
-bigdl/cpp/libs/baby-llama,sha256=QmIJJJGP4U67Wv53jCoYZ9K5g8274o1xGnVRcEVv-J8,14319136
-bigdl/cpp/libs/batched,sha256=CKSDgeHKKPOW4XL-FaUZ4mwfeC1yxs7UH7OtdL3ppB0,14239608
-bigdl/cpp/libs/batched-bench,sha256=w5v59iFWf7oXGSV8BxkhgnFFpp0xTQOUe0u5GcwNq_4,14243864
-bigdl/cpp/libs/benchmark,sha256=xPE6tU_r0tntVYLIYWlR4BNyrmC7bqiulRNLX4ICl-0,1452712
-bigdl/cpp/libs/embedding,sha256=iQcvBMHBhJN00YCmEvUsU3lF3as4SEIYP8H6DapaG9g,14239752
-bigdl/cpp/libs/gguf,sha256=MyC23QGbnIPHNL1YRNqKlRw3KXiyWheq-xsrsmss2Ew,12483008
-bigdl/cpp/libs/imatrix,sha256=XK46doyLDeYNodB2R2wUmyumXDCELebGcqWZLXPr7Fg,14267608
-bigdl/cpp/libs/llama-bench,sha256=jDc6f1ydLVr3gj9DT9dOeYC3LkSZoEOpp_3tu2LH7sw,14333248
-bigdl/cpp/libs/llava-cli,sha256=2oDDOXd50FyhoFyZtfEt8lV25KI0D6vptMMLVNN8z5Q,14557832
-bigdl/cpp/libs/lookahead,sha256=KK9PJ_81kq76j_yNqdgCgtNcLVSVouF6onRrslT4Arg,14252072
-bigdl/cpp/libs/lookup,sha256=igthfHlx4cUXQuIWCMaeUolcV9ilWWuTr3y0M2RHyiI,14271856
-bigdl/cpp/libs/ls-sycl-device,sha256=UtoLLkcRSXWZMDIZs77PAz80_nn19JjSpOZDfMpWZj0,12478752
-bigdl/cpp/libs/main,sha256=h8e9VBzysX0EWA0OQ9Z3rB5Zu6rjaX5GAaoAgj1nAFc,14299192
-bigdl/cpp/libs/ollama,sha256=6CKRaRDw6g9uzO6l5oJKBBsrccqFugw3eJQY20PwdtQ,47375392
-bigdl/cpp/libs/perplexity,sha256=a8eRfIMVI0inrZDWDhxVGfM3L3Yj4ebMlhDbnAbQ4ZI,14340696
-bigdl/cpp/libs/quantize,sha256=RQHpkY72tcZTuvMwOXwEC8fCivnN4ClbUGruj5cKx-g,14265984
-bigdl/cpp/libs/quantize-stats,sha256=YcQ-X5kP8nEPnE0mxZ9Sy1vl_HkjVH_sSiqWJW7YMJc,13780224
-bigdl/cpp/libs/save-load-state,sha256=xekWUPF-IhoNmgxUXSzKMPfadqLNae7iAh1YPQDT9Bc,14239880
-bigdl/cpp/libs/server,sha256=KfI2-m5lAUcxYb6KHVZkXIOy4Kk1VnZ8rLneF5MCCNw,15097272
-bigdl/cpp/libs/speculative,sha256=nCKmu33RQdCwkb5WHaPqfEFvy9RoYnPJHzXi_gLkxT0,14278112
-bigdl/cpp/libs/tokenize,sha256=m9zbFNldBToN00fuM2lQTq1_THx28-HQdFSftw4VNhM,14239800
-bigdl_core_cpp-2.5.0b20240818.data/scripts/init-llama-cpp,sha256=qpIwMQc2Vb11fmovXFJKefp5iD1tQMfL8Ma7Y3Ebguk,400
-bigdl_core_cpp-2.5.0b20240818.data/scripts/init-ollama,sha256=5eg4DsWg87iysFRh7leawr6H47F_sHydwL5we6fUG6o,193
-bigdl_core_cpp-2.5.0b20240818.dist-info/METADATA,sha256=9ccXui2OMJgXwEzx4fJytb_Gi0CCyiiFp6X81l31G14,643
-bigdl_core_cpp-2.5.0b20240818.dist-info/WHEEL,sha256=LP9LuSkp9EkkS3GaxGDqD2GVQk9RkMkUIxzcmUzigkU,108
-bigdl_core_cpp-2.5.0b20240818.dist-info/top_level.txt,sha256=iGuLfZARD_qANcIMfy0tbbrC3EtCg6BSiH8icc3dLWs,6
-bigdl_core_cpp-2.5.0b20240818.dist-info/RECORD,,