llm-benchmark 0.3.1__tar.gz

llm_benchmark-0.3.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 aidatatools (Jason Chuang)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

llm_benchmark-0.3.1/PKG-INFO

@@ -0,0 +1,131 @@
+Metadata-Version: 2.1
+Name: llm_benchmark
+Version: 0.3.1
+Summary: LLM Benchmark for Throughputs via Ollama
+Home-page: https://github.com/aidatatools/ollama-benchmark/
+License: MIT
+Keywords: benchmark,llama,ollama,llms,local
+Author: Jason Chuang
+Author-email: chuangtcee@gmail.com
+Requires-Python: >=3.8,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: gputil (>=1.4.0,<2.0.0)
+Requires-Dist: lib-platform (>=1.2.10,<2.0.0)
+Requires-Dist: ollama (>=0.1.8,<0.2.0)
+Requires-Dist: psutil (>=5.9.8,<6.0.0)
+Requires-Dist: pyyaml (>=6.0.1,<7.0.0)
+Requires-Dist: requests (>=2.31.0,<3.0.0)
+Requires-Dist: typer[all] (>=0.9.0,<0.10.0)
+Description-Content-Type: text/markdown
+
+# ollama-benchmark
+
+LLM Benchmark for Throughput via Ollama (Local LLMs)
+
+## Installation Steps
+
+```bash
+pip3 install ollama-benchmark
+```
+
+It's tested on Python 3.9 and above.
+
+## ollama installation with the following models installed
+
+7B model can be run on machines with 8GB of RAM
+
+13B model can be run on machines with 16GB of RAM
+
+## Usage explaination
+
+On Windows, Linux, and macOS, it will detect memory RAM size to first download required LLM models.
+
+When memory RAM size is greater than or equal to 4GB, but less than 7GB, it will check if gemma:2b exist. The program implicitly pull the model.
+
+```bash
+ollama pull gemma:2b
+```
+
+When memory RAM size is greater than 7GB, but less than 15GB, it will check if these models exist. The program implicitly pull these models
+
+```bash
+ollama pull gemma:2b
+ollama pull gemma:7b
+ollama pull mistral:7b
+ollama pull llama2:7b
+ollama pull llava:7b
+```
+
+When memory RAM siz is greater than 15GB, it will check if these models exist. The program implicitly pull these models
+
+```bash
+ollama pull gemma:2b
+ollama pull gemma:7b
+ollama pull mistral:7b
+ollama pull llama2:7b
+ollama pull llama2:13b
+ollama pull llava:7b
+ollama pull llava:13b
+```
+
+## Usage for general users directly
+
+```bash
+pip install llm-benchmark
+llm_benchmark hello jason
+llm_benchmark run
+```
+
+## Python Poetry manually(advanced) installation
+
+<https://python-poetry.org/docs/#installing-manually>
+
+## For developers to develop new features on Windows Powershell or on Ubuntu Linux or macOS
+
+```bash
+python3 -m venv .venv
+. ./.venv/bin/activate
+pip install -U pip setuptools
+pip install poetry
+```
+
+## Usage in Python virtual environment
+
+```bash
+poetry shell
+poetry install
+llm_benchmark hello jason
+```
+
+### The default sending back the info is
+
+Memory Size: 32GB
+
+CPU: Intel i5-12400
+
+GPU: 3060
+
+OS: Microsoft Windows 11
+
+### Example #1 send systeminfo and bechmark results to a remote server
+
+```bash
+llm_benchmark run
+```
+
+### Example #2 Do not send systeminfo and bechmark results to a remote server
+
+```bash
+llm_benchmark run --no-sendinfo
+```
+
+## Reference
+
+[Ollama](https://ollama.com)
+

llm_benchmark-0.3.1/README.md

@@ -0,0 +1,104 @@
+# ollama-benchmark
+
+LLM Benchmark for Throughput via Ollama (Local LLMs)
+
+## Installation Steps
+
+```bash
+pip3 install ollama-benchmark
+```
+
+It's tested on Python 3.9 and above.
+
+## ollama installation with the following models installed
+
+7B model can be run on machines with 8GB of RAM
+
+13B model can be run on machines with 16GB of RAM
+
+## Usage explaination
+
+On Windows, Linux, and macOS, it will detect memory RAM size to first download required LLM models.
+
+When memory RAM size is greater than or equal to 4GB, but less than 7GB, it will check if gemma:2b exist. The program implicitly pull the model.
+
+```bash
+ollama pull gemma:2b
+```
+
+When memory RAM size is greater than 7GB, but less than 15GB, it will check if these models exist. The program implicitly pull these models
+
+```bash
+ollama pull gemma:2b
+ollama pull gemma:7b
+ollama pull mistral:7b
+ollama pull llama2:7b
+ollama pull llava:7b
+```
+
+When memory RAM siz is greater than 15GB, it will check if these models exist. The program implicitly pull these models
+
+```bash
+ollama pull gemma:2b
+ollama pull gemma:7b
+ollama pull mistral:7b
+ollama pull llama2:7b
+ollama pull llama2:13b
+ollama pull llava:7b
+ollama pull llava:13b
+```
+
+## Usage for general users directly
+
+```bash
+pip install llm-benchmark
+llm_benchmark hello jason
+llm_benchmark run
+```
+
+## Python Poetry manually(advanced) installation
+
+<https://python-poetry.org/docs/#installing-manually>
+
+## For developers to develop new features on Windows Powershell or on Ubuntu Linux or macOS
+
+```bash
+python3 -m venv .venv
+. ./.venv/bin/activate
+pip install -U pip setuptools
+pip install poetry
+```
+
+## Usage in Python virtual environment
+
+```bash
+poetry shell
+poetry install
+llm_benchmark hello jason
+```
+
+### The default sending back the info is
+
+Memory Size: 32GB
+
+CPU: Intel i5-12400
+
+GPU: 3060
+
+OS: Microsoft Windows 11
+
+### Example #1 send systeminfo and bechmark results to a remote server
+
+```bash
+llm_benchmark run
+```
+
+### Example #2 Do not send systeminfo and bechmark results to a remote server
+
+```bash
+llm_benchmark run --no-sendinfo
+```
+
+## Reference
+
+[Ollama](https://ollama.com)

llm_benchmark-0.3.1/llm_benchmark/check_models.py

@@ -0,0 +1,51 @@
+import argparse
+import yaml
+import subprocess
+import ollama
+
+parser = argparse.ArgumentParser(
+    prog="python3 check_models.py",
+    description="Before running check_models.py, please make sure you installed ollama successfully \
+        on macOS, Linux, or WSL2 on Windows. You can check the website: https://ollama.ai",
+    epilog="Author: Jason Chuang")
+
+parser.add_argument("-v",
+                    "--verbose",
+                    action="store_true",
+                    help="this program helps you check whether you have ollama benchmark models installed")
+
+parser.add_argument("-m",
+                    "--models",
+                    type=str,
+                    help="provide benchmark models YAML file path. ex. ../data/benchmark_models.yml")
+
+def parse_yaml(yaml_file_path):
+    with open(yaml_file_path, 'r') as stream:
+        try:
+            data=yaml.safe_load(stream)
+            #print(d)
+        except yaml.YAMLError as e:
+            print(e)
+    return data
+
+def pull_models(models_file_path):
+    print(f"LLM models file path：{models_file_path}")
+    print(f"Checking and pulling the following LLM models")
+    models_dict = parse_yaml(models_file_path)
+    for x in models_dict['models']:
+        model_name = x['model']
+        print(model_name)
+        ollama.pull(model_name)
+
+if __name__ == "__main__": 
+    args = parser.parse_args()
+    #print(f"args.verbose value：{args.verbose}")
+    if (args.models is not None):
+        print(f"args.models file path：{args.models}")
+        models_dict = parse_yaml(args.models)
+        for x in models_dict['models']:
+            model_name = x['model']
+            print(model_name)
+            result = subprocess.run(['ollama', 'pull', model_name], stdout=subprocess.PIPE)
+            result.stdout
+    

llm_benchmark-0.3.1/llm_benchmark/check_ollama.py

@@ -0,0 +1,16 @@
+import subprocess
+
+def run_command(command):
+    try:
+        result = subprocess.run(command, capture_output=True, text=True, check=True)
+        return result.stdout.strip()  # Return the output of the command
+    except subprocess.CalledProcessError as e:
+        print(f"Error executing command '{command}': {e}")
+        return None
+
+def check_ollama_version():
+    res = run_command(['ollama', '--version'])
+    print(res)
+
+if __name__ == "__main__":
+    check_ollama_version()

llm_benchmark-0.3.1/llm_benchmark/main.py

@@ -0,0 +1,85 @@
+import typer, base64
+from llm_benchmark import check_models
+from llm_benchmark import check_ollama
+from llm_benchmark import run_benchmark
+
+from systeminfo import sysmain
+
+from security_connection import connection 
+app = typer.Typer()
+
+
+@app.command()
+def hello(name: str):
+    print(f"Hello {name}!")
+    
+
+@app.command()
+def run(sendinfo: bool = True):
+    sys_info = sysmain.get_extra()
+    print(f"Total memory size : {sys_info['memory']:.2f} GB") 
+    print(f"cpu_info: {sys_info['cpu']}")
+    print(f"gpu_info: {sys_info['gpu']}")
+    print(f"os_version: {sys_info['os_version']}")
+
+    check_ollama.check_ollama_version()
+    print('-'*10)
+
+    ft_mem_size = float(f"{sys_info['memory']:.2f}")
+    models_file_path = 'data/benchmark_models_16gb_ram.yml'
+    if(ft_mem_size>=4 and ft_mem_size <7):
+        models_file_path = 'data/benchmark_models_4gb_ram.yml'
+    elif(ft_mem_size>=7 and ft_mem_size <15):
+        models_file_path = 'data/benchmark_models_8gb_ram.yml'
+
+    check_models.pull_models(models_file_path)
+    print('-'*10)
+
+    benchmark_file_path = 'data/benchmark1.yml'
+
+    bench_results_info = {}
+    result1 = run_benchmark.run_benchmark(models_file_path,benchmark_file_path, 'instruct')
+    bench_results_info.update(result1)
+    result2 = run_benchmark.run_benchmark(models_file_path,benchmark_file_path, 'question-answer')
+    bench_results_info.update(result2)
+    result3 = run_benchmark.run_benchmark(models_file_path,benchmark_file_path, 'vision-image')
+    bench_results_info.update(result3)
+
+    if (sendinfo==True):
+        print(f"Sending the following data to a remote server")
+        print(f"Your machine UUID : {sysmain.get_uuid()}")
+        print(f"{bench_results_info.items()}")
+        print('=='*10)
+        #print(f"{sys_info.items()}")
+        #sys_info = sysmain.get_extra()
+        x = connection.send_sysinfo(sys_info)
+        print(x)
+
+
+@app.command()
+def goodbye(name: str, formal: bool = False):
+    if formal:
+        print(f"Goodbye Mr.(Ms.) {name}. Have a good day.")
+    else:
+        print(f"Bye {name}!")
+
+@app.command()
+def sysinfo(formal: bool = True):
+    if formal:
+        sys_info = sysmain.get_extra()
+        sys_info['uuid'] = f"{sysmain.get_uuid()}"
+        #print(sys_info.items())
+        print(f"memory : {sys_info['memory']:.2f} GB") 
+        print(f"cpu_info: {sys_info['cpu']}")
+        print(f"gpu_info: {sys_info['gpu']}")
+        print(f"os_version: {sys_info['os_version']}")
+        print(f"Your machine UUID : {sys_info['uuid']}")
+
+        x = connection.send_sysinfo(sys_info)
+        print(x)
+    else:
+        print(f"No print!")
+
+
+if __name__ == "__main__":
+    app()

llm_benchmark-0.3.1/llm_benchmark/query_llm.py

@@ -0,0 +1,50 @@
+import requests
+
+headers = {
+    'Content-Type': 'application/json',
+}
+#"model" : "phi"
+#"model" : "llama2"
+#"model" : "mistral"
+
+data = '{\n  "model": "llama2:7b",\n  "prompt": "Why is the sky blue? Is it related to sunshine?",\n  "stream": false\n}'
+
+response = requests.post('http://localhost:11434/api/generate', headers=headers, data=data)
+jsonResponse = response.json()
+
+model = ""
+total_duration = 0.0
+load_duration = 0.0
+prompt_eval_count = 0
+prompt_eval_duration = 0.0
+eval_count = 0
+eval_duration = 0.0
+for key, value in jsonResponse.items():
+    if (key=="response"):
+        pass
+    elif (key=="context"):
+        pass
+    elif (key=="model"):
+        model = value
+    elif (key=="total_duration"):
+        total_duration = float(value)/(10**6)
+    elif (key=="load_duration"):
+        load_duration = float(value)/(10**6)
+    elif (key=="prompt_eval_count"):
+        prompt_eval_count=int(value)
+    elif (key=="prompt_eval_duration"):
+        prompt_eval_duration=float(value)/(10**6)
+    elif (key=="eval_count"):
+        eval_count=int(value)
+    elif (key=="eval_duration"):
+        eval_duration=float(value)/(10**6)
+
+print(f"model = {model}")
+
+print(f"{'total_duration time': >20} = {total_duration:10.2f} ms")
+print(f"{'load_duration time': >20} = {load_duration:10.2f} ms")
+
+print(f"{'prompt eval time ': >20} = {prompt_eval_duration:10.2f} ms / {prompt_eval_count:>6} tokens")    
+print(f"{'eval time ': >20} = {eval_duration:10.2f} ms / {eval_count:>6} tokens ")
+print(f"Performance: {eval_count/eval_duration*1000:10.2f}(tokens/s)")    
+         

llm_benchmark-0.3.1/llm_benchmark/run_benchmark.py

@@ -0,0 +1,117 @@
+import argparse
+import yaml
+import subprocess
+import datetime
+
+parser = argparse.ArgumentParser(
+    prog="python3 check_models.py",
+    description="Before running check_models.py, please make sure you installed ollama successfully \
+        on macOS, Linux, or WSL2 on Windows. You can check the website: https://ollama.com",
+    epilog="Author: Jason Chuang")
+
+parser.add_argument("-v",
+                    "--verbose",
+                    action="store_true",
+                    help="this program helps you check whether you have ollama benchmark models installed")
+
+parser.add_argument("-m",
+                    "--models",
+                    type=str,
+                    help="provide benchmark models YAML file path. ex. ../data/benchmark_models.yml")
+
+parser.add_argument("-b",
+                    "--benchmark",
+                    type=str,
+                    help="provide benchmark YAML file path. ex. ../data/benchmark1.yml")
+
+parser.add_argument("-t",
+                    "--type",
+                    type=str,
+                    help="provide benchmark model type. ex, instruct")
+
+
+def parse_yaml(yaml_file_path):
+    with open(yaml_file_path, 'r') as stream:
+        try:
+            data=yaml.safe_load(stream)
+            #print(d)
+        except yaml.YAMLError as e:
+            print(e)
+    return data
+
+def run_benchmark(models_file_path, benchmark_file_path, type):
+    
+    models_dict = parse_yaml(models_file_path)
+    benchmark_dict = parse_yaml(benchmark_file_path)
+    model_type = type
+    allowed_models = {e['model'] for e in models_dict['models']}
+    #print(allowed_models)
+    ans={}
+    # Writing to file        
+    for one_model_type in benchmark_dict['modeltypes']:
+        #print(one_model_type)
+        if (one_model_type['type']==model_type):
+            #print(one_model_type['models'])
+            #print(one_model_type['prompts'])
+            for onemodel in one_model_type['models']:
+                if (onemodel['model'] in allowed_models ):
+                    loc_dt = datetime.datetime.today()
+                    with open(f'log_{loc_dt.strftime("%Y-%m-%d-%H%M%S")}.log', "w", encoding='utf-8') as file1:
+                        #print(onemodel)
+                        stored_nums=[]
+                        model_name = onemodel['model']
+                        print(f'model_name =    {model_name}')
+                        file1.write(f'\nmodel_name =    {model_name}\n')
+                        
+                        if model_name.startswith('llava'):
+                            for one_prompt in one_model_type['prompts']:
+                                img_file_names = one_prompt['keywords'].split(',')
+                                for img in img_file_names:
+                                    prompt = f"{one_prompt['prompt']} 'data/img/{img}' "
+                                    print(f"prompt = {prompt}")
+                                    result = subprocess.run(['ollama', 'run', model_name, one_prompt['prompt'],'--verbose'], capture_output=True, text=True, check=True, encoding='utf-8')
+                                    std_err = result.stderr
+                                    #print(result.stderr)
+                                    file1.write(std_err)
+                                    
+                                    for line in std_err.split('\n'):
+                                        if ('eval rate' in line) and ('prompt' not in line):
+                                            print(line)
+                                            number = float(line[-20:-8])
+                                            stored_nums.append(number)
+                                            #print(number)
+                        else:
+                            for one_prompt in one_model_type['prompts']:
+                                print(f"prompt = {one_prompt['prompt']}")
+                                result = subprocess.run(['ollama', 'run', model_name, one_prompt['prompt'],'--verbose'], capture_output=True, text=True, check=True, encoding='utf-8')
+                                std_err = result.stderr
+                                #print(result.stderr)                                   
+                                file1.write(std_err)                                    
+                                
+                                for line in std_err.split('\n'):
+                                    if ('eval rate' in line) and ('prompt' not in line):
+                                        print(line)
+                                        number = float(line[-20:-8])
+                                        stored_nums.append(number)
+                                        #print(number)
+
+                        print("-"*20) 
+                        if(len(stored_nums)!=0):       
+                            average = sum(stored_nums)/len(stored_nums)
+                            print("Average of eval rate: ", round(average,3), " tokens/s")
+                            ans[f"{model_name}"]=f"{round(average,3):.2f}"
+
+                        print("-"*40)
+                        file1.write("\n"+"-"*40)
+                    file1.close()
+                    
+    return ans
+
+if __name__ == "__main__": 
+    args = parser.parse_args()
+    #print(f"args.verbose value：{args.verbose}")
+    if (args.models is not None) and (args.benchmark is not None) and (args.type is not None):
+        run_benchmark(args.models, args.benchmark, args.type)
+        print('-'*40)
+        
+        

llm_benchmark-0.3.1/llm_benchmark/run_benchmark_one.py

@@ -0,0 +1,59 @@
+import argparse
+import yaml
+import subprocess
+import datetime
+
+parser = argparse.ArgumentParser(
+    prog="python3 check_models.py",
+    description="Before running check_models.py, please make sure you installed ollama successfully \
+        on macOS, Linux, or WSL2 on Windows. You can check the website: https://ollama.ai",
+    epilog="Author: Jason Chuang")
+
+parser.add_argument("-v",
+                    "--verbose",
+                    action="store_true",
+                    help="this program helps you check whether you have ollama benchmark models installed")
+
+parser.add_argument("-m",
+                    "--models",
+                    type=str,
+                    help="provide benchmark models YAML file path. ex. ../data/benchmark_models.yml")
+
+def parse_yaml(yaml_file_path):
+    with open(yaml_file_path, 'r') as stream:
+        try:
+            data=yaml.safe_load(stream)
+            #print(d)
+        except yaml.YAMLError as e:
+            print(e)
+    return data
+
+if __name__ == "__main__": 
+    args = parser.parse_args()
+    #print(f"args.verbose value：{args.verbose}")
+    if (args.models is not None):
+        #print(f"args.models file path：{args.models}")
+        models_dict = parse_yaml(args.models)
+        
+        loc_dt = datetime.datetime.today()
+        # Writing to file
+        with open(f'log_{loc_dt.strftime("%Y-%m-%d-%H%M%S")}.log', "w") as file1:
+            for onemodel in models_dict['models']:
+                model_name = onemodel['model']
+                print(f'model_name =    {model_name}')
+                file1.write(f'\nmodel_name =    {model_name}\n')
+                
+                result = subprocess.run(['ollama', 'run', model_name,f'"Why is the sky blue?"','--verbose'], capture_output=True, text=True, check=True)
+                std_err = result.stderr
+                #print(result.stderr)
+                file1.write(std_err)
+                
+                for line in std_err.split('\n'):
+                    if ('eval rate' in line) and ('prompt' not in line):
+                        print(line)
+                        number = float(line[-20:-8])
+                        print(number)
+                        
+                
+                print("-"*40)
+                file1.write("\n"+"-"*40)

llm_benchmark-0.3.1/pyproject.toml

@@ -0,0 +1,35 @@
+[tool.poetry]
+name = "llm_benchmark"
+version = "0.3.1"
+description = "LLM Benchmark for Throughputs via Ollama"
+authors = ["Jason Chuang <chuangtcee@gmail.com>"]
+license = "MIT"
+homepage = "https://github.com/aidatatools/ollama-benchmark/"
+readme = "README.md"
+keywords = [
+    "benchmark",
+    "llama",
+    "ollama",
+    "llms",
+    "local",
+]
+
+[tool.poetry.dependencies]
+python = "^3.8"
+typer = {extras = ["all"], version = "^0.9.0"}
+ollama = "^0.1.8"
+pyyaml = "^6.0.1"
+requests = "^2.31.0"
+psutil = "^5.9.8"
+gputil = "^1.4.0"
+lib-platform = "^1.2.10"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.1.1"
+
+[tool.poetry.scripts]
+llm_benchmark = "llm_benchmark.main:app"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"