ailia-llm 1.2.0.0__tar.gz

ailia_llm-1.2.0.0/PKG-INFO

@@ -0,0 +1,78 @@
+Metadata-Version: 2.1
+Name: ailia_llm
+Version: 1.2.0.0
+Summary: ailia LLM
+Home-page: https://ailia.jp/
+Author: ax Inc.
+Author-email: contact@axinc.jp
+License: https://ailia.ai/en/license/
+Requires-Python: >3.6
+Description-Content-Type: text/markdown
+Requires-Dist: ailia
+
+# ailia LLM Python API
+
+!! CAUTION !!
+“ailia” IS NOT OPEN SOURCE SOFTWARE (OSS).
+As long as user complies with the conditions stated in [License Document](https://ailia.ai/license/), user may use the Software for free of charge, but the Software is basically paid software.
+
+## About ailia LLM
+
+ailia LLM is a library for running LLMs on edge devices. It provides bindings for C++ and Unity.
+
+## Install from pip
+
+You can install the ailia LLM free evaluation package with the following command.
+
+```
+pip3 install ailia_llm
+```
+
+## Install from package
+
+You can install the ailia LLM from Package with the following command.
+
+```
+python3 bootstrap.py
+pip3 install ./
+```
+
+## Usage
+
+```python
+import ailia_llm
+
+import os
+import urllib.request
+
+model_file_path = "gemma-2-2b-it-Q4_K_M.gguf"
+if not os.path.exists(model_file_path):
+	urllib.request.urlretrieve(
+		"https://storage.googleapis.com/ailia-models/gemma/gemma-2-2b-it-Q4_K_M.gguf",
+		model_file_path
+	)
+
+model = ailia_llm.AiliaLLM()
+model.open(model_file_path)
+
+messages = []
+messages.append({"role": "system", "content": "語尾に「わん」をつけてください。"})
+messages.append({"role": "user", "content": "あなたの名前は何ですか？"})
+
+stream = model.generate(messages)
+
+text = ""
+for delta_text in stream:
+	text = text + delta_text
+print(text)
+
+if model.context_full():
+	raise Exception("Context full")
+
+messages.append({"role": "assistant", "content": text})
+```
+
+## API specification
+
+https://github.com/axinc-ai/ailia-sdk
+

ailia_llm-1.2.0.0/README.md

@@ -0,0 +1,66 @@
+# ailia LLM Python API
+
+!! CAUTION !!
+“ailia” IS NOT OPEN SOURCE SOFTWARE (OSS).
+As long as user complies with the conditions stated in [License Document](https://ailia.ai/license/), user may use the Software for free of charge, but the Software is basically paid software.
+
+## About ailia LLM
+
+ailia LLM is a library for running LLMs on edge devices. It provides bindings for C++ and Unity.
+
+## Install from pip
+
+You can install the ailia LLM free evaluation package with the following command.
+
+```
+pip3 install ailia_llm
+```
+
+## Install from package
+
+You can install the ailia LLM from Package with the following command.
+
+```
+python3 bootstrap.py
+pip3 install ./
+```
+
+## Usage
+
+```python
+import ailia_llm
+
+import os
+import urllib.request
+
+model_file_path = "gemma-2-2b-it-Q4_K_M.gguf"
+if not os.path.exists(model_file_path):
+	urllib.request.urlretrieve(
+		"https://storage.googleapis.com/ailia-models/gemma/gemma-2-2b-it-Q4_K_M.gguf",
+		model_file_path
+	)
+
+model = ailia_llm.AiliaLLM()
+model.open(model_file_path)
+
+messages = []
+messages.append({"role": "system", "content": "語尾に「わん」をつけてください。"})
+messages.append({"role": "user", "content": "あなたの名前は何ですか？"})
+
+stream = model.generate(messages)
+
+text = ""
+for delta_text in stream:
+	text = text + delta_text
+print(text)
+
+if model.context_full():
+	raise Exception("Context full")
+
+messages.append({"role": "assistant", "content": text})
+```
+
+## API specification
+
+https://github.com/axinc-ai/ailia-sdk
+

ailia_llm-1.2.0.0/ailia_llm/__init__.py

@@ -0,0 +1,201 @@
+import ctypes
+import os
+import sys
+
+import numpy
+import ailia
+
+import os
+import platform
+
+#### dependency check
+if sys.platform == "win32":
+    import ctypes
+    try:
+        for library in ["vcruntime140.dll", "vcruntime140_1.dll", "msvcp140.dll"]:
+            ctypes.windll.LoadLibrary(library)
+    except:
+        print("  WARNING Please install MSVC 2015-2019 runtime from https://docs.microsoft.com/ja-jp/cpp/windows/latest-supported-vc-redist")
+
+
+#### loading DLL / DYLIB / SO  ####
+if sys.platform == "win32":
+    dll_platform = "windows/x64"
+    dll_name = "ailia_llm.dll"
+    dll_name_fallback = "ailia_llm_fallback.dll"
+    load_fn = ctypes.WinDLL
+elif sys.platform == "darwin":
+    dll_platform = "mac"
+    dll_name = "libailia_llm.dylib"
+    dll_name_fallback = None
+    load_fn = ctypes.CDLL
+else:
+    is_arm = "arm" in platform.machine() or platform.machine() == "aarch64"
+    if is_arm:
+        if platform.architecture()[0] == "32bit":
+            dll_platform = "linux/armeabi-v7a"
+        else:
+            dll_platform = "linux/arm64-v8a"
+    else:
+        dll_platform = "linux/x64"
+    dll_name = "libailia_llm.so"
+    dll_name_fallback = None
+    load_fn = ctypes.CDLL
+
+dll_found = False
+candidate = ["", str(os.path.dirname(os.path.abspath(__file__))) + str(os.sep), str(os.path.dirname(os.path.abspath(__file__))) + str(os.sep) + dll_platform + str(os.sep)]
+
+for dir in candidate:
+    try:
+        dll = load_fn(dir + dll_name) # Load gpu library first
+        dll_found = True
+    except:
+        pass
+
+if not dll_found:
+    for dir in candidate:
+        try:
+            dll = load_fn(dir + dll_name_fallback) # Fallback cpu library second
+            dll_found = True
+        except:
+            pass
+
+if not dll_found:
+    msg = "DLL load failed : \'" + dll_name + "\' is not found"
+    raise ImportError(msg)
+
+# ==============================================================================
+
+from ctypes import *
+
+AILIA_LLM_STATUS_SUCCESS = ( 0 )
+AILIA_LLM_STATUS_CONTEXT_FULL = ( -8 )
+
+class AILIALLMChatMessage(ctypes.Structure):
+    _fields_ = [
+        ("role", ctypes.c_char_p),
+        ("content", ctypes.c_char_p)
+    ]
+
+# ==============================================================================
+
+dll.ailiaLLMCreate.restype = c_int
+dll.ailiaLLMCreate.argtypes = (POINTER(c_void_p), )
+
+dll.ailiaLLMDestroy.restype = None
+dll.ailiaLLMDestroy.argtypes = (c_void_p, )
+
+dll.ailiaLLMOpenModelFileA.restype = c_int
+dll.ailiaLLMOpenModelFileA.argtypes = (c_void_p, c_char_p, c_uint32)
+
+dll.ailiaLLMOpenModelFileW.restype = c_int
+dll.ailiaLLMOpenModelFileW.argtypes = (c_void_p, c_wchar_p, c_uint32)
+
+dll.ailiaLLMGetDeltaTextSize.restype = c_int
+dll.ailiaLLMGetDeltaTextSize.argtypes = (c_void_p, POINTER(c_uint32))
+
+dll.ailiaLLMGetDeltaText.restype = c_int
+dll.ailiaLLMGetDeltaText.argtypes = (c_void_p, numpy.ctypeslib.ndpointer(
+                dtype=numpy.uint8, flags='CONTIGUOUS'
+            ),                               # text
+            ctypes.c_uint)
+
+dll.ailiaLLMSetPrompt.restype = c_int
+dll.ailiaLLMSetPrompt.argtypes = (c_void_p, ctypes.POINTER(AILIALLMChatMessage), ctypes.c_uint)
+
+dll.ailiaLLMGenerate.restype = c_int
+dll.ailiaLLMGenerate.argtypes = (c_void_p, POINTER(c_uint32))
+
+# ==============================================================================
+# base model class
+# ==============================================================================
+
+class AiliaLLMError(RuntimeError):
+    def __init__(self, message, code):
+        super().__init__(f"{message} code:{code}")
+        self.code = code
+
+class AiliaLLM():
+    _instance = None
+    _context_full = False
+
+    def _check(self, status):
+        if status != AILIA_LLM_STATUS_SUCCESS:
+            raise AiliaLLMError(f"ailia LLM error", status)
+
+    def _string_buffer_aw(self, path):
+        if sys.platform == "win32":
+            return ctypes.create_unicode_buffer(path)
+        else:
+            return ctypes.create_string_buffer(path.encode("utf-8"))
+
+    def _string_buffer(self, path):
+        return ctypes.create_string_buffer(path.encode("utf-8"))
+
+    def _string_to_c_char_p(self, s):
+        return ctypes.c_char_p(s.encode('utf-8'))
+
+    def __init__(self):
+        self._instance = ctypes.c_void_p(None)
+        self._check(dll.ailiaLLMCreate(cast(pointer(self._instance), POINTER(c_void_p))))
+
+    def open(self, model_path, n_ctx = 0):
+        if "time_license" in ailia.get_version():
+            ailia.check_and_download_license()
+        p1 = self._string_buffer_aw(model_path)
+
+        if sys.platform == "win32":
+            self._check(dll.ailiaLLMOpenModelFileW(self._instance, p1, ctypes.c_uint32(n_ctx)))
+        else:
+            self._check(dll.ailiaLLMOpenModelFileA(self._instance, p1, ctypes.c_uint32(n_ctx)))
+
+    def generate(self, prompts):
+        messages = (AILIALLMChatMessage * len(prompts))()
+        for i in range(len(prompts)):
+            messages[i].role = self._string_to_c_char_p(prompts[i]["role"])
+            messages[i].content = self._string_to_c_char_p(prompts[i]["content"])
+
+        self._context_full = False
+        status = dll.ailiaLLMSetPrompt(self._instance, messages, ctypes.c_uint32(len(prompts)))
+        if status == AILIA_LLM_STATUS_CONTEXT_FULL:
+            self._context_full = True
+            return
+        self._check(status)
+
+        buf = numpy.zeros((0), dtype=numpy.uint8, order='C')
+        before_text = ""
+
+        while True:
+            done = ctypes.c_uint(0)
+            status = dll.ailiaLLMGenerate(self._instance, ctypes.byref(done))
+            self._check(status)
+
+            if status == AILIA_LLM_STATUS_CONTEXT_FULL:
+                self._context_full = True
+                break
+
+            if done.value == 1:
+                break
+
+            count = ctypes.c_uint(0)
+            self._check(dll.ailiaLLMGetDeltaTextSize(self._instance, ctypes.byref(count)))
+
+            new_buf = numpy.zeros((count.value), dtype=numpy.uint8, order='C')
+
+            self._check(dll.ailiaLLMGetDeltaText(self._instance, new_buf, count))
+
+            buf = numpy.concatenate([buf[:-1], new_buf])
+
+            text = bytes(buf[:- 1]).decode("utf-8", errors="ignore")
+            delta_text = text[len(before_text):]
+
+            yield delta_text
+            before_text = text
+
+    def context_full(self):
+        return self._context_full
+
+    def __del__(self):
+        if self._instance:
+            dll.ailiaLLMDestroy(cast(self._instance, c_void_p))
+

ailia_llm-1.2.0.0/ailia_llm/oss/LICENSE_LLAMA_CPP.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023-2024 The ggml authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ailia_llm-1.2.0.0/ailia_llm.egg-info/PKG-INFO

@@ -0,0 +1,78 @@
+Metadata-Version: 2.1
+Name: ailia_llm
+Version: 1.2.0.0
+Summary: ailia LLM
+Home-page: https://ailia.jp/
+Author: ax Inc.
+Author-email: contact@axinc.jp
+License: https://ailia.ai/en/license/
+Requires-Python: >3.6
+Description-Content-Type: text/markdown
+Requires-Dist: ailia
+
+# ailia LLM Python API
+
+!! CAUTION !!
+“ailia” IS NOT OPEN SOURCE SOFTWARE (OSS).
+As long as user complies with the conditions stated in [License Document](https://ailia.ai/license/), user may use the Software for free of charge, but the Software is basically paid software.
+
+## About ailia LLM
+
+ailia LLM is a library for running LLMs on edge devices. It provides bindings for C++ and Unity.
+
+## Install from pip
+
+You can install the ailia LLM free evaluation package with the following command.
+
+```
+pip3 install ailia_llm
+```
+
+## Install from package
+
+You can install the ailia LLM from Package with the following command.
+
+```
+python3 bootstrap.py
+pip3 install ./
+```
+
+## Usage
+
+```python
+import ailia_llm
+
+import os
+import urllib.request
+
+model_file_path = "gemma-2-2b-it-Q4_K_M.gguf"
+if not os.path.exists(model_file_path):
+	urllib.request.urlretrieve(
+		"https://storage.googleapis.com/ailia-models/gemma/gemma-2-2b-it-Q4_K_M.gguf",
+		model_file_path
+	)
+
+model = ailia_llm.AiliaLLM()
+model.open(model_file_path)
+
+messages = []
+messages.append({"role": "system", "content": "語尾に「わん」をつけてください。"})
+messages.append({"role": "user", "content": "あなたの名前は何ですか？"})
+
+stream = model.generate(messages)
+
+text = ""
+for delta_text in stream:
+	text = text + delta_text
+print(text)
+
+if model.context_full():
+	raise Exception("Context full")
+
+messages.append({"role": "assistant", "content": text})
+```
+
+## API specification
+
+https://github.com/axinc-ai/ailia-sdk
+

ailia_llm-1.2.0.0/ailia_llm.egg-info/SOURCES.txt

@@ -0,0 +1,17 @@
+README.md
+setup.py
+ailia_llm/LICENSE_AILIA_EN.pdf
+ailia_llm/LICENSE_AILIA_JA.pdf
+ailia_llm/__init__.py
+ailia_llm.egg-info/PKG-INFO
+ailia_llm.egg-info/SOURCES.txt
+ailia_llm.egg-info/dependency_links.txt
+ailia_llm.egg-info/requires.txt
+ailia_llm.egg-info/top_level.txt
+ailia_llm/linux/x64/libailia_llm.so
+ailia_llm/mac/libailia_llm.dylib
+ailia_llm/oss/LICENSE_LLAMA_CPP.txt
+ailia_llm/windows/x64/ailia_llm.dll
+ailia_llm/windows/x64/ailia_llm_fallback.dll
+ailia_llm/windows/x64/ggml.dll
+ailia_llm/windows/x64/llama.dll

ailia_llm-1.2.0.0/ailia_llm.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

ailia_llm-1.2.0.0/ailia_llm.egg-info/requires.txt

@@ -0,0 +1 @@
+ailia

ailia_llm-1.2.0.0/ailia_llm.egg-info/top_level.txt

@@ -0,0 +1 @@
+ailia_llm

ailia_llm-1.2.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

ailia_llm-1.2.0.0/setup.py

@@ -0,0 +1,70 @@
+import os
+import sys
+import platform
+import glob
+import shutil
+import platform
+
+from setuptools import setup, Extension
+from setuptools import find_packages
+
+with open("README.md", encoding="utf-8") as f:
+    long_description = f.read()
+
+scripts = []
+for f in glob.glob("ailia_llm/*.py"):
+    scripts.append(f)
+
+def find_libraries():
+    dll_names = []
+    platforms = ["win32", "darwin", "linux_armv7l", "linux_aarch64", "linux_x86_64"]
+
+    for platform in platforms:
+        if platform == "win32":
+            dll_platform = "windows/x64"
+            dll_type = ".dll"
+        elif platform == "darwin":
+            dll_platform = "mac"
+            dll_type = ".dylib"
+        else:
+            if platform == "linux_armv7l":
+                dll_platform = "linux/armeabi-v7a"
+            elif platform == "linux_aarch64":
+                dll_platform = "linux/arm64-v8a"
+            else:
+                dll_platform = "linux/x64"
+            dll_type = ".so"
+    
+        dll_path = "./ailia_llm/" + dll_platform + "/"
+
+        for f in glob.glob(dll_path+"*"+dll_type):
+            f = f.replace("\\", "/")
+            f = f.replace("./ailia_llm/", "./")
+            dll_names.append(f)
+
+    dll_names.append("./LICENSE_AILIA_EN.pdf")
+    dll_names.append("./LICENSE_AILIA_JA.pdf")
+    dll_names.append("./oss/LICENSE_LLAMA_CPP.txt")
+
+    return dll_names
+    
+if __name__ == "__main__":
+    setup(
+        name="ailia_llm",
+        scripts=scripts,
+        version="1.2.0.0",
+        install_requires=[
+            "ailia",
+        ],
+        description="ailia LLM",
+        long_description=long_description,
+        long_description_content_type="text/markdown",
+        author="ax Inc.",
+        author_email="contact@axinc.jp",
+        url="https://ailia.jp/",
+        license="https://ailia.ai/en/license/",
+        packages=find_packages(),
+        package_data={"ailia_llm":find_libraries()},
+        include_package_data=True,
+        python_requires=">3.6",
+    )