bigdl-core-cpp 2.5.0b20240817__py3-none-manylinux2010_x86_64.whl → 2.5.0b20240819__py3-none-manylinux2010_x86_64.whl

bigdl/cpp/convert-hf-to-gguf.py

@@ -1570,6 +1570,33 @@ class LlamaModel(Model):
         return [(self.map_tensor_name(name), data_torch)]
 
     def prepare_tensors(self):
+        if rope_scaling := self.find_hparam(["rope_scaling"], optional=True):
+            if rope_scaling.get("rope_type", '').lower() == "llama3":
+                base = self.hparams.get("rope_theta", 10000.0)
+                dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"]
+                freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
+                factor = rope_scaling.get("factor", 8.0)
+                low_freq_factor = rope_scaling.get("low_freq_factor", 1.0)
+                high_freq_factor = rope_scaling.get("high_freq_factor", 4.0)
+                old_context_len = self.hparams.get("original_max_position_embeddings", 8192)
+
+                low_freq_wavelen = old_context_len / low_freq_factor
+                high_freq_wavelen = old_context_len / high_freq_factor
+                assert low_freq_wavelen != high_freq_wavelen
+
+                rope_factors = []
+                for freq in freqs:
+                    wavelen = 2 * math.pi / freq
+                    if wavelen < high_freq_wavelen:
+                        rope_factors.append(1)
+                    elif wavelen > low_freq_wavelen:
+                        rope_factors.append(factor)
+                    else:
+                        smooth = (old_context_len / wavelen - low_freq_factor) / (high_freq_factor - low_freq_factor)
+                        rope_factors.append(1 / ((1 - smooth) / factor + smooth))
+
+                self.gguf_writer.add_tensor(self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), np.array(rope_factors, dtype=np.float32))
+
         super().prepare_tensors()
 
         if self._experts is not None:
@@ -3686,4 +3713,4 @@ def main() -> None:
 
 
 if __name__ == '__main__':
-    main()
+    main()

{bigdl_core_cpp-2.5.0b20240817.dist-info → bigdl_core_cpp-2.5.0b20240819.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bigdl-core-cpp
-Version: 2.5.0b20240817
+Version: 2.5.0b20240819
 Summary: Large Language Model Develop Toolkit
 Author: BigDL Authors
 License: Apache License, Version 2.0

bigdl_core_cpp-2.5.0b20240819.dist-info/RECORD

@@ -0,0 +1,45 @@
+bigdl/cpp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bigdl/cpp/convert-hf-to-gguf.py,sha256=ci596u_GY7MDZrPeUGJ3TnYYzdZ2_1e0JT-nLeNpps8,168262
+bigdl/cpp/convert.py,sha256=XMMcpfWHwEAAWzwLXe9mmJTU7cMvcyw8g2BFctfZnvI,69417
+bigdl/cpp/cli/init-llama-cpp,sha256=qpIwMQc2Vb11fmovXFJKefp5iD1tQMfL8Ma7Y3Ebguk,400
+bigdl/cpp/cli/init-ollama,sha256=5eg4DsWg87iysFRh7leawr6H47F_sHydwL5we6fUG6o,193
+bigdl/cpp/gguf-py/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bigdl/cpp/gguf-py/gguf/__init__.py,sha256=LUfJeRbrLY6TgwI0rLw_qo8klu7GgYB8Ycu8H4V0eb8,218
+bigdl/cpp/gguf-py/gguf/constants.py,sha256=WRloarpYU4sc0tIi8aDgIh6xb6Ox0CPLCiPa9Yzzm2Q,46902
+bigdl/cpp/gguf-py/gguf/gguf.py,sha256=MzR6GNwyrWBN3w-o995FY0qX8U3J9Mbinu9HAMEjx8g,477
+bigdl/cpp/gguf-py/gguf/gguf_reader.py,sha256=5yv15RSKLfz9ViQd2e-vQxzXEdCGqgJfGfoBbzPYTHw,12364
+bigdl/cpp/gguf-py/gguf/gguf_writer.py,sha256=HqxGAplbhGDgn9bELLFFh3Nf15Gg16BOTVvhOMKe6GM,34612
+bigdl/cpp/gguf-py/gguf/lazy.py,sha256=yMcMTIrlwpQrX9m8YSRdvyHsNupHZMSj1RsjAAPrxqE,8527
+bigdl/cpp/gguf-py/gguf/metadata.py,sha256=HNu6fzuLLSzQ_K_xJkKqtT5Tni3kGESj67BZ2uPNajg,24851
+bigdl/cpp/gguf-py/gguf/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+bigdl/cpp/gguf-py/gguf/quants.py,sha256=dI6JsSpaSzfdz7OwfqH1kvK98M_Ax8d6NMNYY5pyoQM,4354
+bigdl/cpp/gguf-py/gguf/tensor_mapping.py,sha256=1tNoM-XXvD8JaStxt39HBRe_obCWawHfTFVCWXRe9Zk,30147
+bigdl/cpp/gguf-py/gguf/utility.py,sha256=xEIfT3CphqAyI56A5XZQDMti2XGG5k6kxm5bjf_Vn-E,2933
+bigdl/cpp/gguf-py/gguf/vocab.py,sha256=vsRrPQ6RafirhBmMp9h3YGZh_tqEjeg7S2hiLvMv7Ys,6847
+bigdl/cpp/libs/baby-llama,sha256=kGnqhEAwmNjOaSON9snxmR4GAq_5HCriisvPJGPamCU,14319136
+bigdl/cpp/libs/batched,sha256=aK7ZX2JgPRLTDHxIaleEvsG3AbMHjvxFXKP9WUfp-NM,14243704
+bigdl/cpp/libs/batched-bench,sha256=cjr9xVgw4f9wh-NDpD9NklxJ7Xso_47B3KUS1V5EelU,14243864
+bigdl/cpp/libs/benchmark,sha256=JWcWqfNXjZKVgOECu0SLOnwP3rQhpv_YfVp1QKEBidg,1452712
+bigdl/cpp/libs/embedding,sha256=qA_brIrkGSsoTf6Qy1ZMKV_98_SOaQdsDYGtkc40Y4k,14239752
+bigdl/cpp/libs/gguf,sha256=cwv9Ztq7yDdE8iZWubcJyqsdtOhJ7mDys3KAIQWnm3g,12483008
+bigdl/cpp/libs/imatrix,sha256=-0OXkkyNgx5LDLbTG_VZmgBk__tlWIZczvYOoR_3EQA,14267608
+bigdl/cpp/libs/llama-bench,sha256=McOBvMpMx7BMbiJhheGHlqWBt_IAEwMYK3WrLV5gsu8,14333248
+bigdl/cpp/libs/llava-cli,sha256=32SrWle5wkKlT56rxjQHiM87aE2lYVUD_6ZN6BbI90o,14557832
+bigdl/cpp/libs/lookahead,sha256=ZRFg1C3hETHD7pJNEizNWdYVuS1xRwpcbrPZUxiH7B8,14252072
+bigdl/cpp/libs/lookup,sha256=ADm7SA7TLQi7eY5cf8EYDNCSskgetY0z5KLkscb-puU,14271856
+bigdl/cpp/libs/ls-sycl-device,sha256=6qJNKOzfRlI4zCdDMNFb-tkBCnFhLa8i7xrINUrqXQg,12478752
+bigdl/cpp/libs/main,sha256=nAii7sWRlgcVDLhQ5uTyNW9m48-2ag3-bGX4L1m4Ovw,14299192
+bigdl/cpp/libs/ollama,sha256=T9T0LKZbKRtZduTkYeaHwDdd-9lAmKWWp3R15FrMcwA,47374520
+bigdl/cpp/libs/perplexity,sha256=xrfGT763Fsag9CWKII2N9CqeVBovkcg0iUFBcd-9fe0,14340696
+bigdl/cpp/libs/quantize,sha256=Gj7RQHVADwmxLuQFAezNWntQyw_hQvciUmgOncApYqk,14265984
+bigdl/cpp/libs/quantize-stats,sha256=bYXwOFOrsNrpEIPdBP-L0NBNCFHrFYJFFiHndsSIGlQ,13780224
+bigdl/cpp/libs/save-load-state,sha256=N4txBmZdicwhlABupzBy8bwtJFsyjNdxIE1UB7BVDlU,14239880
+bigdl/cpp/libs/server,sha256=7wKqNYgjpTV6Iq_LfjgT3VMk4BOvv8vWs2S8p7Jmnp4,15097272
+bigdl/cpp/libs/speculative,sha256=UtePoo5dm9V0oU5fvCQ0azLe-Xwtngm_1Qdif1a0f0k,14278112
+bigdl/cpp/libs/tokenize,sha256=hBgsuXhJReBXePKG2TW2jjbm6bg3GzfCjihyCq7rPYo,14239800
+bigdl_core_cpp-2.5.0b20240819.data/scripts/init-llama-cpp,sha256=qpIwMQc2Vb11fmovXFJKefp5iD1tQMfL8Ma7Y3Ebguk,400
+bigdl_core_cpp-2.5.0b20240819.data/scripts/init-ollama,sha256=5eg4DsWg87iysFRh7leawr6H47F_sHydwL5we6fUG6o,193
+bigdl_core_cpp-2.5.0b20240819.dist-info/METADATA,sha256=OXzNuN0jo3su5UibesviWusnKuhMweT8Yf5CtN0Cf30,643
+bigdl_core_cpp-2.5.0b20240819.dist-info/WHEEL,sha256=LP9LuSkp9EkkS3GaxGDqD2GVQk9RkMkUIxzcmUzigkU,108
+bigdl_core_cpp-2.5.0b20240819.dist-info/top_level.txt,sha256=iGuLfZARD_qANcIMfy0tbbrC3EtCg6BSiH8icc3dLWs,6
+bigdl_core_cpp-2.5.0b20240819.dist-info/RECORD,,

bigdl_core_cpp-2.5.0b20240817.dist-info/RECORD

@@ -1,45 +0,0 @@
-bigdl/cpp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bigdl/cpp/convert-hf-to-gguf.py,sha256=1wAaTiqtAsnA_4k10TCzDSp4KTZ_mldv6pY47z5gg7M,166672
-bigdl/cpp/convert.py,sha256=XMMcpfWHwEAAWzwLXe9mmJTU7cMvcyw8g2BFctfZnvI,69417
-bigdl/cpp/cli/init-llama-cpp,sha256=qpIwMQc2Vb11fmovXFJKefp5iD1tQMfL8Ma7Y3Ebguk,400
-bigdl/cpp/cli/init-ollama,sha256=5eg4DsWg87iysFRh7leawr6H47F_sHydwL5we6fUG6o,193
-bigdl/cpp/gguf-py/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bigdl/cpp/gguf-py/gguf/__init__.py,sha256=LUfJeRbrLY6TgwI0rLw_qo8klu7GgYB8Ycu8H4V0eb8,218
-bigdl/cpp/gguf-py/gguf/constants.py,sha256=WRloarpYU4sc0tIi8aDgIh6xb6Ox0CPLCiPa9Yzzm2Q,46902
-bigdl/cpp/gguf-py/gguf/gguf.py,sha256=MzR6GNwyrWBN3w-o995FY0qX8U3J9Mbinu9HAMEjx8g,477
-bigdl/cpp/gguf-py/gguf/gguf_reader.py,sha256=5yv15RSKLfz9ViQd2e-vQxzXEdCGqgJfGfoBbzPYTHw,12364
-bigdl/cpp/gguf-py/gguf/gguf_writer.py,sha256=HqxGAplbhGDgn9bELLFFh3Nf15Gg16BOTVvhOMKe6GM,34612
-bigdl/cpp/gguf-py/gguf/lazy.py,sha256=yMcMTIrlwpQrX9m8YSRdvyHsNupHZMSj1RsjAAPrxqE,8527
-bigdl/cpp/gguf-py/gguf/metadata.py,sha256=HNu6fzuLLSzQ_K_xJkKqtT5Tni3kGESj67BZ2uPNajg,24851
-bigdl/cpp/gguf-py/gguf/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bigdl/cpp/gguf-py/gguf/quants.py,sha256=dI6JsSpaSzfdz7OwfqH1kvK98M_Ax8d6NMNYY5pyoQM,4354
-bigdl/cpp/gguf-py/gguf/tensor_mapping.py,sha256=1tNoM-XXvD8JaStxt39HBRe_obCWawHfTFVCWXRe9Zk,30147
-bigdl/cpp/gguf-py/gguf/utility.py,sha256=xEIfT3CphqAyI56A5XZQDMti2XGG5k6kxm5bjf_Vn-E,2933
-bigdl/cpp/gguf-py/gguf/vocab.py,sha256=vsRrPQ6RafirhBmMp9h3YGZh_tqEjeg7S2hiLvMv7Ys,6847
-bigdl/cpp/libs/baby-llama,sha256=M2eD-B0Qf8Y7b0PtQE1mS12-LxHt1l8QoNIQC8jAx6A,14319136
-bigdl/cpp/libs/batched,sha256=bAUzmgdw1uqXWl8ZaqMzxbYDrF9Ufo3Jw_8fM6jBhwM,14239608
-bigdl/cpp/libs/batched-bench,sha256=_0YqfDgZ0YQQPr8juykKLzFQSl2jOjLDRN8excMRH1k,14243864
-bigdl/cpp/libs/benchmark,sha256=UwDZLdWvpHeeIB6YXhEj8fz78COGMld6jZm_V6zkdKE,1452712
-bigdl/cpp/libs/embedding,sha256=z2BoCONRuNK73jwzmOwmvwQ14pwU5dtwXUQNurvkPpw,14239752
-bigdl/cpp/libs/gguf,sha256=_8q73hMRwpnsChxwcKYfv9Z12sFoLMixnnhuiZDlqgU,12483008
-bigdl/cpp/libs/imatrix,sha256=qKZFNCz416HSm0vKlyIrm8G3EftzzqOFnKhsjf4JrBQ,14267608
-bigdl/cpp/libs/llama-bench,sha256=FLQl6ClQBoXgDTfk-Bsn73hxi4PXKtV8cpWzLMtJGO4,14333248
-bigdl/cpp/libs/llava-cli,sha256=m--CY_PRRuWHcxnHafJL08yOPyIb9jFHtLRiEMKrU0o,14557832
-bigdl/cpp/libs/lookahead,sha256=T7te888UT5ZC7lpZELT5NHMRWe7HAlQEHOxzP_i6HfU,14252072
-bigdl/cpp/libs/lookup,sha256=AfkhRZoSp0dK8VIdED9JFPJTVZJPuBK1sbEduLqzl8Y,14271856
-bigdl/cpp/libs/ls-sycl-device,sha256=xbER8DNMx3_I8VIefd1c0uUGeWNeB3-B7JFX3ST3E6M,12478752
-bigdl/cpp/libs/main,sha256=x9jt9M8d767KJYlBQ-DSSHM2GNcI3WtlZQbBk0y6DCg,14299192
-bigdl/cpp/libs/ollama,sha256=2zMXwamY_Lam-gG0XORFZDfD2itTlWnVSE-k-rNlHiY,47375392
-bigdl/cpp/libs/perplexity,sha256=V28BvHIBuCzXlC9dWExqJ8U2SvG2aaDoBw6WE5ZkHd0,14340696
-bigdl/cpp/libs/quantize,sha256=wwRsWxrZwF1UFLHtc--CdrS9QJyLVJ2w_HUgYEcE5dQ,14265984
-bigdl/cpp/libs/quantize-stats,sha256=Q8hSEpU8IY2zyQz2ncDbTNIAA9CYbO5f2tK1WgeRuhM,13780224
-bigdl/cpp/libs/save-load-state,sha256=1oCehQolHU334yFQ0hTxPAVVT-oPBsoSeccfqOQKFsw,14239880
-bigdl/cpp/libs/server,sha256=0Dud9bfkr3YvibOLC6eahgTzKdMdspS2IOhOqBxXC0c,15097272
-bigdl/cpp/libs/speculative,sha256=Tm-TECoRyWUXzKOOjnWxxBREiLcDVfEQMjPtn1RqxFA,14278112
-bigdl/cpp/libs/tokenize,sha256=oN-QrflOawfucpEeXOC1tXO44kN-17Kt7PPuX21jWSY,14239800
-bigdl_core_cpp-2.5.0b20240817.data/scripts/init-llama-cpp,sha256=qpIwMQc2Vb11fmovXFJKefp5iD1tQMfL8Ma7Y3Ebguk,400
-bigdl_core_cpp-2.5.0b20240817.data/scripts/init-ollama,sha256=5eg4DsWg87iysFRh7leawr6H47F_sHydwL5we6fUG6o,193
-bigdl_core_cpp-2.5.0b20240817.dist-info/METADATA,sha256=7B9qEJppl_IbxoojYhKB60s0bIEfgmvj0c5CryCu4os,643
-bigdl_core_cpp-2.5.0b20240817.dist-info/WHEEL,sha256=LP9LuSkp9EkkS3GaxGDqD2GVQk9RkMkUIxzcmUzigkU,108
-bigdl_core_cpp-2.5.0b20240817.dist-info/top_level.txt,sha256=iGuLfZARD_qANcIMfy0tbbrC3EtCg6BSiH8icc3dLWs,6
-bigdl_core_cpp-2.5.0b20240817.dist-info/RECORD,,