llm-benchmark 0.3.27__py2.py3-none-any.whl

llm_benchmark/check_models.py

@@ -0,0 +1,51 @@
+import argparse
+import yaml
+import subprocess
+import ollama
+
+parser = argparse.ArgumentParser(
+    prog="python3 check_models.py",
+    description="Before running check_models.py, please make sure you installed ollama successfully \
+        on macOS, Linux, or WSL2 on Windows. You can check the website: https://ollama.ai",
+    epilog="Author: Jason Chuang")
+
+parser.add_argument("-v",
+                    "--verbose",
+                    action="store_true",
+                    help="this program helps you check whether you have ollama benchmark models installed")
+
+parser.add_argument("-m",
+                    "--models",
+                    type=str,
+                    help="provide benchmark models YAML file path. ex. ../data/benchmark_models.yml")
+
+def parse_yaml(yaml_file_path):
+    with open(yaml_file_path, 'r') as stream:
+        try:
+            data=yaml.safe_load(stream)
+            #print(d)
+        except yaml.YAMLError as e:
+            print(e)
+    return data
+
+def pull_models(models_file_path):
+    print(f"LLM models file path：{models_file_path}")
+    print(f"Checking and pulling the following LLM models")
+    models_dict = parse_yaml(models_file_path)
+    for x in models_dict['models']:
+        model_name = x['model']
+        print(model_name)
+        ollama.pull(model_name)
+
+if __name__ == "__main__": 
+    args = parser.parse_args()
+    #print(f"args.verbose value：{args.verbose}")
+    if (args.models is not None):
+        print(f"args.models file path：{args.models}")
+        models_dict = parse_yaml(args.models)
+        for x in models_dict['models']:
+            model_name = x['model']
+            print(model_name)
+            result = subprocess.run(['ollama', 'pull', model_name], stdout=subprocess.PIPE)
+            result.stdout
+    

llm_benchmark/check_ollama.py

@@ -0,0 +1,22 @@
+import subprocess
+
+def run_command(command):
+    try:
+        result = subprocess.run(command, capture_output=True, text=True, check=True)
+        return result.stdout.strip()  # Return the output of the command
+    except subprocess.CalledProcessError as e:
+        print(f"Error executing command '{command}': {e}")
+        return None
+
+def check_ollama_version(ollamabin='ollama'):
+    res = run_command([ollamabin, '--version'])
+    ans = res.split('\n')
+    #print(ans[-1])
+    if("warning" in ans[-1].lower()):
+        return ans[-1][27:]
+    else:
+        return ans[-1][18:]
+
+if __name__ == "__main__":
+    check_ollama_version()
+    check_ollama_version('ollama')

llm_benchmark/data/benchmark1.yml

@@ -0,0 +1,57 @@
+# Author: Jason Chuang
+# License: MIT
+version: 1.0
+modeltypes:
+  - type: chat
+    prompts:
+      - prompt: What are your thoughts on the latest scientific discovery regarding black holes?
+        keywords: science, recent
+  - type: completion
+    prompts:
+      - prompt: In a world where time travel is possible, a scientist invents a device that allows people to visit any historical event. One day, a young woman decides to travel back in time to...
+        keywords:
+  - type: instruct
+    models:
+      - model: mistral:7b
+      - model: llama3:8b
+      - model: llama3.1:8b
+      - model: phi4:14b
+      - model: qwen:1.8b
+      - model: qwen2:7b
+      - model: phi:2.7b
+    prompts:
+      - prompt: Write a step-by-step guide on how to bake a chocolate cake from scratch.
+        keywords: cooking, recipe
+      - prompt: Develop a python function that solves the following problem, sudoku game
+        keywords: python, sudoku
+      - prompt: Create a dialogue between two characters that discusses economic crisis
+        keywords: dialogue
+      - prompt: In a forest, there are brave lions living there. Please continue the story.
+        keywords: sentence completition
+      - prompt: I'd like to book a flight for 4 to Seattle in U.S.
+        keywords: flight booking
+  - type: question-answer
+    reference_url: https://www.turing.com/interview-questions/artificial-intelligence
+    models:
+      - model: gemma:2b
+      - model: gemma:7b
+      - model: gemma2:9b
+    prompts:
+      - prompt: Explain Artificial Intelligence and give its applications.
+        keywords:
+      - prompt: How are machine learning and AI related?
+        keywords:
+      - prompt: What is Deep Learning based on?
+        keywords:
+      - prompt: What is the full form of LSTM?
+        keywords:
+      - prompt: What are different components of GAN?
+        keywords:
+  - type: vision-image
+    reference_url: https://chuangtc.com/Research/llm-vlm.php
+    models:
+      - model: llava:7b
+      - model: llava:13b
+    prompts:
+      - prompt: Describe the image,
+        keywords: sample1.jpg,sample2.jpg,sample3.jpg,sample4.jpg,sample5.jpg

llm_benchmark/data/benchmark_models_16gb_ram.yml

@@ -0,0 +1,11 @@
+# Author: Jason Chuang
+# License: MIT
+version: 1.0
+models:
+  - model: phi4:14b
+  - model: qwen2:7b
+  - model: gemma2:9b
+  - model: mistral:7b
+  - model: llama3.1:8b
+  - model: llava:7b
+  - model: llava:13b

llm_benchmark/data/benchmark_models_2gb_ram.yml

@@ -0,0 +1,6 @@
+# Author: Jason Chuang
+# License: MIT
+version: 1.0
+models:
+  - model: qwen2:1.5b
+  - model: gemma:2b

llm_benchmark/data/benchmark_models_3gb_ram.yml

@@ -0,0 +1,7 @@
+# Author: Jason Chuang
+# License: MIT
+version: 1.0
+models:
+  - model: qwen2:1.5b
+  - model: gemma:2b
+  - model: phi:2.7b

llm_benchmark/data/benchmark_models_4gb_ram.yml

@@ -0,0 +1,8 @@
+# Author: Jason Chuang
+# License: MIT
+version: 1.0
+models:
+  - model: qwen2:1.5b
+  - model: gemma:2b
+  - model: phi:2.7b
+  - model: phi3:3.8b

llm_benchmark/data/benchmark_models_8gb_ram.yml

@@ -0,0 +1,10 @@
+# Author: Jason Chuang
+# License: MIT
+version: 1.0
+models:
+  - model: phi3:3.8b
+  - model: qwen2:7b
+  - model: gemma2:9b
+  - model: mistral:7b
+  - model: llama3.1:8b
+  - model: llava:7b

llm_benchmark/main.py

@@ -0,0 +1,101 @@
+import typer, base64
+from llm_benchmark import check_models
+from llm_benchmark import check_ollama
+from llm_benchmark import run_benchmark
+
+from .systeminfo import sysmain
+
+from .security_connection import connection 
+
+import pkg_resources
+
+
+app = typer.Typer()
+
+@app.command()
+def hello(name: str):
+    print(f"Hello {name}!")
+    
+
+@app.command()
+def run(ollamabin: str = 'ollama' , sendinfo : bool = True ):
+    sys_info = sysmain.get_extra()
+    print(f"Total memory size : {sys_info['memory']:.2f} GB") 
+    print(f"cpu_info: {sys_info['cpu']}")
+    print(f"gpu_info: {sys_info['gpu']}")
+    print(f"os_version: {sys_info['os_version']}")
+
+    ollama_version = check_ollama.check_ollama_version(ollamabin)
+    print(f"ollama_version: {ollama_version}")
+    print('-'*10)
+
+    ft_mem_size = float(f"{sys_info['memory']:.2f}")
+    models_file_path = pkg_resources.resource_filename('llm_benchmark','data/benchmark_models_16gb_ram.yml')
+    if(ft_mem_size>=1 and ft_mem_size <2):
+        models_file_path = pkg_resources.resource_filename('llm_benchmark','data/benchmark_models_2gb_ram.yml')
+    elif(ft_mem_size>=2 and ft_mem_size <4):
+        models_file_path = pkg_resources.resource_filename('llm_benchmark','data/benchmark_models_3gb_ram.yml')
+    elif(ft_mem_size>=4 and ft_mem_size <7):
+        models_file_path = pkg_resources.resource_filename('llm_benchmark','data/benchmark_models_4gb_ram.yml')
+    elif(ft_mem_size>=7 and ft_mem_size <15):
+        models_file_path = pkg_resources.resource_filename('llm_benchmark','data/benchmark_models_8gb_ram.yml')
+
+    check_models.pull_models(models_file_path)
+    print('-'*10)
+
+    benchmark_file_path = pkg_resources.resource_filename('llm_benchmark','data/benchmark1.yml')
+
+    bench_results_info = {}
+    is_simulation = False
+    if is_simulation==False :
+        result1 = run_benchmark.run_benchmark(models_file_path,benchmark_file_path, 'instruct', ollamabin)
+        bench_results_info.update(result1)
+        result2 = run_benchmark.run_benchmark(models_file_path,benchmark_file_path, 'question-answer', ollamabin)
+        bench_results_info.update(result2)
+        result3 = run_benchmark.run_benchmark(models_file_path,benchmark_file_path, 'vision-image', ollamabin)
+        bench_results_info.update(result3)
+    else:
+        bench_results_info.update({"llama2:7b":7.65})
+        bench_results_info.update({"gemma2:7b":17.77})
+
+    if (sendinfo==True):
+        print(f"Sending the following data to a remote server")
+        print(f"Your machine UUID : {sysmain.get_uuid()}")
+        #print(f"{bench_results_info.items()}")
+        x = connection.send_benchmark(sysmain.get_uuid(),ollama_version,bench_results_info)
+        #print(x)
+        print('=='*10)
+        #print(f"{sys_info.items()}")
+        sys_info = sysmain.get_extra()
+        sys_info['uuid']=f"{sysmain.get_uuid()}"
+        x = connection.send_sysinfo(sys_info)
+        #print(x)
+
+
+@app.command()
+def goodbye(name: str, formal: bool = False):
+    if formal:
+        print(f"Goodbye Mr.(Ms.) {name}. Have a good day.")
+    else:
+        print(f"Bye {name}!")
+
+@app.command()
+def sysinfo(formal: bool = True):
+    if formal:
+        sys_info = sysmain.get_extra()
+        sys_info['uuid'] = f"{sysmain.get_uuid()}"
+        #print(sys_info.items())
+        print(f"memory : {sys_info['memory']:.2f} GB") 
+        print(f"cpu_info: {sys_info['cpu']}")
+        print(f"gpu_info: {sys_info['gpu']}")
+        print(f"os_version: {sys_info['os_version']}")
+        print(f"Your machine UUID : {sys_info['uuid']}")
+
+        x = connection.send_sysinfo(sys_info)
+        print(x)
+    else:
+        print(f"No print!")
+
+
+if __name__ == "__main__":
+    app()

llm_benchmark/query_llm.py

@@ -0,0 +1,50 @@
+import requests
+
+headers = {
+    'Content-Type': 'application/json',
+}
+#"model" : "phi"
+#"model" : "llama2"
+#"model" : "mistral"
+
+data = '{\n  "model": "llama2:7b",\n  "prompt": "Why is the sky blue? Is it related to sunshine?",\n  "stream": false\n}'
+
+response = requests.post('http://localhost:11434/api/generate', headers=headers, data=data)
+jsonResponse = response.json()
+
+model = ""
+total_duration = 0.0
+load_duration = 0.0
+prompt_eval_count = 0
+prompt_eval_duration = 0.0
+eval_count = 0
+eval_duration = 0.0
+for key, value in jsonResponse.items():
+    if (key=="response"):
+        pass
+    elif (key=="context"):
+        pass
+    elif (key=="model"):
+        model = value
+    elif (key=="total_duration"):
+        total_duration = float(value)/(10**6)
+    elif (key=="load_duration"):
+        load_duration = float(value)/(10**6)
+    elif (key=="prompt_eval_count"):
+        prompt_eval_count=int(value)
+    elif (key=="prompt_eval_duration"):
+        prompt_eval_duration=float(value)/(10**6)
+    elif (key=="eval_count"):
+        eval_count=int(value)
+    elif (key=="eval_duration"):
+        eval_duration=float(value)/(10**6)
+
+print(f"model = {model}")
+
+print(f"{'total_duration time': >20} = {total_duration:10.2f} ms")
+print(f"{'load_duration time': >20} = {load_duration:10.2f} ms")
+
+print(f"{'prompt eval time ': >20} = {prompt_eval_duration:10.2f} ms / {prompt_eval_count:>6} tokens")    
+print(f"{'eval time ': >20} = {eval_duration:10.2f} ms / {eval_count:>6} tokens ")
+print(f"Performance: {eval_count/eval_duration*1000:10.2f}(tokens/s)")    
+         

llm_benchmark/run_benchmark.py

@@ -0,0 +1,119 @@
+import argparse
+import yaml
+import subprocess
+import datetime
+import pkg_resources
+
+parser = argparse.ArgumentParser(
+    prog="python3 check_models.py",
+    description="Before running check_models.py, please make sure you installed ollama successfully \
+        on macOS, Linux, or WSL2 on Windows. You can check the website: https://ollama.com",
+    epilog="Author: Jason Chuang")
+
+parser.add_argument("-v",
+                    "--verbose",
+                    action="store_true",
+                    help="this program helps you check whether you have ollama benchmark models installed")
+
+parser.add_argument("-m",
+                    "--models",
+                    type=str,
+                    help="provide benchmark models YAML file path. ex. ../data/benchmark_models.yml")
+
+parser.add_argument("-b",
+                    "--benchmark",
+                    type=str,
+                    help="provide benchmark YAML file path. ex. ../data/benchmark1.yml")
+
+parser.add_argument("-t",
+                    "--type",
+                    type=str,
+                    help="provide benchmark model type. ex, instruct")
+
+
+def parse_yaml(yaml_file_path):
+    with open(yaml_file_path, 'r') as stream:
+        try:
+            data=yaml.safe_load(stream)
+            #print(d)
+        except yaml.YAMLError as e:
+            print(e)
+    return data
+
+def run_benchmark(models_file_path, benchmark_file_path, type, ollamabin):
+    
+    models_dict = parse_yaml(models_file_path)
+    benchmark_dict = parse_yaml(benchmark_file_path)
+    model_type = type
+    allowed_models = {e['model'] for e in models_dict['models']}
+    #print(allowed_models)
+    ans={}
+    # Writing to file        
+    for one_model_type in benchmark_dict['modeltypes']:
+        #print(one_model_type)
+        if (one_model_type['type']==model_type):
+            #print(one_model_type['models'])
+            #print(one_model_type['prompts'])
+            for onemodel in one_model_type['models']:
+                if (onemodel['model'] in allowed_models ):
+                    loc_dt = datetime.datetime.today()
+                    with open(f'log_{loc_dt.strftime("%Y-%m-%d-%H%M%S")}.log', "w", encoding='utf-8') as file1:
+                        #print(onemodel)
+                        stored_nums=[]
+                        model_name = onemodel['model']
+                        print(f'model_name =    {model_name}')
+                        file1.write(f'\nmodel_name =    {model_name}\n')
+                        
+                        if model_name.startswith('llava'):
+                            for one_prompt in one_model_type['prompts']:
+                                img_file_names = one_prompt['keywords'].split(',')
+                                for img in img_file_names:
+                                    img_file_path = pkg_resources.resource_filename('llm_benchmark',f'data/img/{img}')
+                                    prompt = f"{one_prompt['prompt']} {img_file_path}"
+                                    print(f"prompt = {prompt}")
+                                    result = subprocess.run([ollamabin, 'run', model_name, one_prompt['prompt'],'--verbose'], capture_output=True, text=True, check=True, encoding='utf-8')
+                                    std_err = result.stderr
+                                    #print(result.stderr)
+                                    file1.write(std_err)
+                                    
+                                    for line in std_err.split('\n'):
+                                        if ('eval rate' in line) and ('prompt' not in line):
+                                            print(line)
+                                            number = float(line[-20:-8])
+                                            stored_nums.append(number)
+                                            #print(number)
+                        else:
+                            for one_prompt in one_model_type['prompts']:
+                                print(f"prompt = {one_prompt['prompt']}")
+                                result = subprocess.run([ollamabin, 'run', model_name, one_prompt['prompt'],'--verbose'], capture_output=True, text=True, check=True, encoding='utf-8')
+                                std_err = result.stderr
+                                #print(result.stderr)                                   
+                                file1.write(std_err)                                    
+                                
+                                for line in std_err.split('\n'):
+                                    if ('eval rate' in line) and ('prompt' not in line):
+                                        print(line)
+                                        number = float(line[-20:-8])
+                                        stored_nums.append(number)
+                                        #print(number)
+
+                        print("-"*20) 
+                        if(len(stored_nums)!=0):       
+                            average = sum(stored_nums)/len(stored_nums)
+                            print("Average of eval rate: ", round(average,3), " tokens/s")
+                            ans[f"{model_name}"]=f"{round(average,3):.2f}"
+
+                        print("-"*40)
+                        file1.write("\n"+"-"*40)
+                    file1.close()
+                    
+    return ans
+
+if __name__ == "__main__": 
+    args = parser.parse_args()
+    #print(f"args.verbose value：{args.verbose}")
+    if (args.models is not None) and (args.benchmark is not None) and (args.type is not None):
+        run_benchmark(args.models, args.benchmark, args.type)
+        print('-'*40)
+        
+        

llm_benchmark/run_benchmark_one.py

@@ -0,0 +1,59 @@
+import argparse
+import yaml
+import subprocess
+import datetime
+
+parser = argparse.ArgumentParser(
+    prog="python3 check_models.py",
+    description="Before running check_models.py, please make sure you installed ollama successfully \
+        on macOS, Linux, or WSL2 on Windows. You can check the website: https://ollama.ai",
+    epilog="Author: Jason Chuang")
+
+parser.add_argument("-v",
+                    "--verbose",
+                    action="store_true",
+                    help="this program helps you check whether you have ollama benchmark models installed")
+
+parser.add_argument("-m",
+                    "--models",
+                    type=str,
+                    help="provide benchmark models YAML file path. ex. ../data/benchmark_models.yml")
+
+def parse_yaml(yaml_file_path):
+    with open(yaml_file_path, 'r') as stream:
+        try:
+            data=yaml.safe_load(stream)
+            #print(d)
+        except yaml.YAMLError as e:
+            print(e)
+    return data
+
+if __name__ == "__main__": 
+    args = parser.parse_args()
+    #print(f"args.verbose value：{args.verbose}")
+    if (args.models is not None):
+        #print(f"args.models file path：{args.models}")
+        models_dict = parse_yaml(args.models)
+        
+        loc_dt = datetime.datetime.today()
+        # Writing to file
+        with open(f'log_{loc_dt.strftime("%Y-%m-%d-%H%M%S")}.log', "w") as file1:
+            for onemodel in models_dict['models']:
+                model_name = onemodel['model']
+                print(f'model_name =    {model_name}')
+                file1.write(f'\nmodel_name =    {model_name}\n')
+                
+                result = subprocess.run(['ollama', 'run', model_name,f'"Why is the sky blue?"','--verbose'], capture_output=True, text=True, check=True)
+                std_err = result.stderr
+                #print(result.stderr)
+                file1.write(std_err)
+                
+                for line in std_err.split('\n'):
+                    if ('eval rate' in line) and ('prompt' not in line):
+                        print(line)
+                        number = float(line[-20:-8])
+                        print(number)
+                        
+                
+                print("-"*40)
+                file1.write("\n"+"-"*40)

llm_benchmark/security_connection/connection.py

@@ -0,0 +1,30 @@
+import requests,json
+
+
+remote_server = "http://llm.aidatatools.com"
+#emote_server = "http://localhost:5000"
+
+def send_sysinfo(data_dict):
+    
+    json_object = json.dumps(data_dict, indent = 4)
+    print(json_object)
+    print('-'*10)
+
+    url = f'{remote_server}/api/receiver.php'
+
+    x = requests.post(url, data = json_object)
+
+    return x.text 
+
+def send_benchmark(uuid, ollama_version,data_dict):
+    data_dict.update({"uuid":f"{uuid}"})
+    data_dict.update({"ollama_version":f"{ollama_version}"})
+    json_object = json.dumps(data_dict, indent = 4)
+    print(json_object)
+    print('-'*10)
+
+    url = f'{remote_server}/api/receiver_benchmark.php'
+
+    x = requests.post(url, data = json_object)
+
+    return x.text 

llm_benchmark/systeminfo/sysmain.py

@@ -0,0 +1,242 @@
+import platform
+import psutil
+import GPUtil
+import subprocess
+import os
+import uuid
+import base64
+
+def get_total_memory_size():
+    memory_info = psutil.virtual_memory()
+    return (memory_info.total/(2**(10*3)))
+
+def get_system_info():
+    #print("System Information:")
+    system_info = platform.uname()
+    ans={}
+    ans['system'] = f"{system_info.system}"
+    ans['node_name'] = f"{system_info.node}"
+    ans['release'] = f"{system_info.release}"
+    ans['version'] = f"{system_info.version}"
+    ans['machine'] = f"{system_info.machine}"
+    ans['processor'] = f"{system_info.processor}"
+    return ans
+
+def get_cpu_info():
+    cpu_info = platform.processor()
+    cpu_count = psutil.cpu_count(logical=False)
+    logical_cpu_count = psutil.cpu_count(logical=True)
+
+    #print("\nCPU Information:")
+    ans = {}
+    ans['processor'] = f"{cpu_info}"
+    ans['physical_cores'] = f"{cpu_count}"
+    ans['logical_cores'] = f"{logical_cpu_count}"
+    return ans
+
+def get_memory_info():
+    memory_info = psutil.virtual_memory()
+
+    #print("\nMemory Information: Unit : bytes")
+    ans={}
+    ans['total_memory'] = f"{memory_info.total}"
+    ans['available_memory'] = f"{memory_info.available}"
+    ans['used_memory'] = f"{memory_info.used}"
+    ans['memory_utilization'] = f"{memory_info.percent:.2f}%"
+    return ans
+
+def get_disk_info():
+    disk_info = psutil.disk_usage('/')
+
+    #print("\nDisk Information: Unit : bytes")
+    ans={}
+    ans['total_disk_space'] = f"{disk_info.total}"
+    ans['used_disk_space'] = f"{disk_info.used}"
+    ans['free_disk_space'] = f"{disk_info.free}"
+    ans['disk_space_utilization'] = f" {disk_info.percent:.2f}%"
+
+#Only Nvidia GPU on Windows and Linux
+def get_gpu_info():    
+
+    ans={}
+    if (get_system_info()['system']=='Darwin'):
+        ans['0'] = "no_gpu"
+        return ans
+    else:
+        gpus = GPUtil.getGPUs()
+    
+    if not gpus:
+        print("\nNo GPU detected.")
+        ans['0'] = "no_gpu"
+    else:
+        ans['0'] = "there_is_gpu"
+
+        for i, gpu in enumerate(gpus):
+            #print(f"\nGPU {i + 1} Information:")
+            ans[f'{i+1}'] = {}
+            ans[f'{i+1}']['id'] = f"{gpu.id}"
+            ans[f'{i+1}']['name'] = f"{' '.join(gpu.name.splitlines())}"
+            ans[f'{i+1}']['driver'] = f"{gpu.driver}"
+            ans[f'{i+1}']['gpu_memory_total'] = f"{gpu.memoryTotal} MB"
+            ans[f'{i+1}']['gpu_memory_free'] = f"{gpu.memoryFree} MB"
+            ans[f'{i+1}']['gpu_memory_used'] = f"{gpu.memoryUsed} MB"
+            ans[f'{i+1}']['gpu_load'] = f"{gpu.load*100}%"
+            ans[f'{i+1}']['gpu_temperature'] = f"{gpu.temperature}°C"
+    
+    return ans
+
+def check_windows_shell():
+    parent_pid = os.getppid()
+    shell_name = psutil.Process(parent_pid).name().lower()
+    if 'cmd' in shell_name:
+        return 'CMD'
+    elif 'powershell' in shell_name:
+        return 'PowerShell'
+    else:
+        return 'Unknown'
+
+
+def get_extra():
+    system_info = platform.uname()
+    ans={}
+    ans['system'] = f"{system_info.system}"
+    ans['memory'] = get_total_memory_size()
+    ans['cpu'] = f"unknown"
+    ans['gpu'] = f"unknown"
+    ans['os_version'] = f"unknown"
+
+    try:
+        if(system_info.system=='Darwin'):
+            ans['system_name'] = "macOS"
+
+            print('----------Apple Mac---------')
+            r1 = subprocess.run(['system_profiler', 'SPHardwareDataType'],capture_output=True,text=True)
+            #ans['hardware'] = f"{r1.stdout}"
+            for line in r1.stdout.split('\n'):
+                if('Model Identifier' in line):
+                    ans['model']=f"{line[24:]}"
+                    
+
+            for line in r1.stdout.split('\n'):
+                if ('Chip' in line):
+                    ans['cpu']=f"{line[12:]}"
+            
+            if(ans['cpu'].startswith('Apple')):
+                ans['gpu'] = ans['cpu']
+            else:
+                ans['gpu'] = 'no_gpu'
+            
+            #r2 = subprocess.run(['system_profiler', 'SPDisplaysDataType'],capture_output=True,text=True)
+            #ans['display'] = f"{r2.stdout}"
+            r3 = subprocess.run(['system_profiler', 'SPSoftwareDataType'],capture_output=True,text=True)
+            #ans['software'] = f"{r3.stdout}"
+
+            for line in r3.stdout.split('\n'):
+                if ('System Version' in line):
+                    ans['os_version']=f"{line[22:]}"
+            return ans
+        elif(system_info.system=='Linux'):
+            ans['system_name'] = "Linux"
+
+            print('-------Linux----------')
+
+            try:     
+                r2 = subprocess.run(['lsb_release','-a'],capture_output=True,text=True)
+                software = f"{r2.stdout}"
+                for line in software.split('\n'):
+                    if ('Description' in line):
+                        ans['os_version']=f"{line[12:]}".strip()
+            except:
+                r2 = subprocess.run(['cat','/etc/os-release'],capture_output=True,text=True)
+                software = f"{r2.stdout}"
+                for line in software.split('\n'):
+                    if ('PRETTY_NAME' in line):
+                        ans['os_version']=f"{line[12:]}".strip()
+            
+            try:
+                r1 = subprocess.run(['lshw','-C','cpu'],capture_output=True,text=True)
+                for line in r1.stdout.split('\n'):
+                    if ('product' in line):
+                        ans['cpu']=f"{line[16:]}"
+            except:
+                cmd = ['lscpu']
+                ps = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+                cmd = ['grep', 'Model name']
+                grep = subprocess.Popen(cmd, stdin=ps.stdout, stdout=subprocess.PIPE,
+                        encoding='utf-8')
+                ps.stdout.close()
+                output, _ = grep.communicate()
+                python_processes = output.split('\n')
+                ans['cpu'] = python_processes[0][16:].strip() 
+
+            if (get_gpu_info()['0']=='no_gpu'):
+                try:
+                    r2 = subprocess.run(['lshw','-C','display'],capture_output=True,text=True)
+                    for line in r2.stdout.split('\n'):
+                        if ('product' in line):
+                            ans['gpu']=f"{line[16:]}"
+                except:
+                    ans['gpu']="no_gpu"
+            else :
+                print(f"{get_gpu_info()['1']}")
+                try:
+                     print(get_gpu_info()['2'])
+                     print('At least two GPU cards')
+                except:
+                     print('Only one GPU card')
+                #List only the first gpu name
+                ans['gpu']=get_gpu_info()['1']['name']
+
+                                        
+            return ans
+            
+        elif(system_info.system=='Windows'):
+            ans['system_name'] = "Windows"
+
+            prefix_exe='powershell.exe'
+            #print("Python is running in:", check_windows_shell(), "on Windows")
+            ans['run_in'] = f"{check_windows_shell()}"           
+                        
+            r1 = subprocess.run([prefix_exe,'Get-WmiObject','Win32_Processor'],capture_output=True,text=True)
+            #print(r1.stdout)
+            r_cpu = subprocess.run([prefix_exe,'(Get-WmiObject Win32_Processor).Name'],capture_output=True,text=True)
+            #print(r_cpu.stdout)
+            ans['cpu']=r_cpu.stdout.strip()
+            r2 = subprocess.run([prefix_exe,'Get-WmiObject','Win32_PhysicalMemory'],capture_output=True,text=True)
+            #print(r2.stdout)
+            r3 = subprocess.run([prefix_exe,'Get-WmiObject','Win32_VideoController'],capture_output=True,text=True)
+            #print(r3.stdout)
+            r_gpu = subprocess.run([prefix_exe,'(Get-WmiObject Win32_VideoController).Caption'],capture_output=True,text=True)
+            str_gpu = r_gpu.stdout.strip()
+            str_gpu = ' '.join(str_gpu.splitlines())
+            #print(r_gpu)
+            ans['gpu'] = str_gpu
+            r4 = subprocess.run([prefix_exe,'(Get-WmiObject Win32_OperatingSystem).Caption'],capture_output=True,text=True)
+            #print(r4.stdout)
+            ans['os_version'] = f"{r4.stdout}"
+        
+        return ans
+
+    except:
+        print("error! when retrieving os_version, cpu, or gpu !")
+    
+    return ans
+
+def get_uuid():
+    sys_info = get_extra()
+    system_name = sys_info['system_name']
+    cpu = sys_info['cpu']
+    gpu  = sys_info['gpu']
+    memory = f"{sys_info['memory']:.2f}"
+
+    id_str = f"{system_name}|{cpu}|{gpu}|{memory}"
+    #print(id_str)
+    uuid5 = uuid.uuid5(uuid.NAMESPACE_X500, id_str)
+    return uuid5    
+
+if __name__ == "__main__":
+    #sysinfo = get_extra()
+    uuid5 = get_uuid()
+    print("UUID version 5:", uuid5)
+    
+

llm_benchmark-0.3.27.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 aidatatools (Jason Chuang)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

llm_benchmark-0.3.27.dist-info/METADATA

@@ -0,0 +1,147 @@
+Metadata-Version: 2.2
+Name: llm_benchmark
+Version: 0.3.27
+Summary: LLM Benchmark
+Home-page: https://github.com/aidatatools/ollama-benchmark
+Author: Jason Chuang
+Author-email: chuangtcee@gmail.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: click>=8.0.0
+Requires-Dist: typer==0.11.0
+Requires-Dist: ollama==0.2.1
+Requires-Dist: pyyaml==6.0.1
+Requires-Dist: requests==2.31.0
+Requires-Dist: psutil==5.9.8
+Requires-Dist: GPUtil==1.4.0
+Requires-Dist: lib-platform==1.2.10
+Requires-Dist: setuptools==69.1.0
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# llm-benchmark (ollama-benchmark)
+
+LLM Benchmark for Throughput via Ollama (Local LLMs)
+
+## Installation prerequisites
+
+Working [Ollama](https://ollama.com) installation.
+
+## Installation Steps
+
+Depending on your python setup either
+```bash
+pip install llm-benchmark
+```
+or
+```bash
+pipx install llm-benchmark
+```
+
+## Usage for general users directly
+
+```bash
+llm_benchmark run
+```
+
+## Installation and Usage in Video format
+
+![llm-benchmark](https://github.com/aidatatools/ollama-benchmark/blob/main/llm-benchmark.gif)
+
+It's tested on Python 3.9 and above.
+
+## ollama installation with the following models installed
+
+7B model can be run on machines with 8GB of RAM
+
+13B model can be run on machines with 16GB of RAM
+
+## Usage explaination
+
+On Windows, Linux, and macOS, it will detect memory RAM size to first download required LLM models.
+
+When memory RAM size is greater than or equal to 4GB, but less than 7GB, it will check if gemma:2b exist. The program implicitly pull the model.
+
+```bash
+ollama pull qwen:1.8b
+ollama pull gemma:2b
+ollama pull phi:2.7b
+ollama pull phi3:3.8b
+```
+
+When memory RAM size is greater than 7GB, but less than 15GB, it will check if these models exist. The program implicitly pull these models
+
+```bash
+ollama pull phi3:3.8b
+ollama pull qwen2:7b
+ollama pull gemma2:9b
+ollama pull mistral:7b
+ollama pull llama3.1:8b
+ollama pull llava:7b
+```
+
+When memory RAM siz is greater than 15GB, it will check if these models exist. The program implicitly pull these models
+
+```bash
+ollama pull phi4:14b
+ollama pull qwen2:7b
+ollama pull gemma2:9b
+ollama pull mistral:7b
+ollama pull llama3.1:8b
+ollama pull llava:7b
+ollama pull llava:13b
+```
+
+## Python Poetry manually(advanced) installation
+
+<https://python-poetry.org/docs/#installing-manually>
+
+## For developers to develop new features on Windows Powershell or on Ubuntu Linux or macOS
+
+```bash
+python3 -m venv .venv
+. ./.venv/bin/activate
+pip install -U pip setuptools
+pip install poetry
+```
+
+## Usage in Python virtual environment
+
+```bash
+poetry shell
+poetry install
+llm_benchmark hello jason
+```
+
+### Example #1 send systeminfo and benchmark results to a remote server
+
+```bash
+llm_benchmark run
+```
+
+### Example #2 Do not send systeminfo and benchmark results to a remote server
+
+```bash
+llm_benchmark run --no-sendinfo
+```
+
+### Example #3 Benchmark run on explicitly given the path to the ollama executable (When you built your own developer version of ollama)
+
+```bash
+llm_benchmark run --ollamabin=~/code/ollama/ollama
+```
+
+## Reference
+
+[Ollama](https://ollama.com)

llm_benchmark-0.3.27.dist-info/RECORD

@@ -0,0 +1,29 @@
+llm_benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+llm_benchmark/check_models.py,sha256=plcAUqp8953j9MFHoAPJHN9AxzhE9pYVjTN7luML_2k,1837
+llm_benchmark/check_ollama.py,sha256=VyMLzWj5O6RjeFeijxFQXjzj0_enf4O-r5bsRnj_AnA,693
+llm_benchmark/main.py,sha256=17Pks6EOSjqzJW_WwqpwtqkYEdQehwTkI_Ka6hoNSR0,3682
+llm_benchmark/query_llm.py,sha256=1g_RZPTnL-rMs0dfBXPlBmUUxutpRG1bCj_R9eSppnY,1623
+llm_benchmark/run_benchmark.py,sha256=CSYgbOogQvwbuDSzNU_H2I2LxHpKv2ZC8sR7Cl70i1I,5797
+llm_benchmark/run_benchmark_one.py,sha256=FSqrdx5wM5f36kggYBlhs6SF9DERaiiRr8TO1Y6FOKk,2338
+llm_benchmark/data/benchmark1.yml,sha256=VEPaCvuD4hl-Ls6Nx4-kk29RliXFccXog-w2wevWwpI,2192
+llm_benchmark/data/benchmark_models_16gb_ram.yml,sha256=vpuh_OvP4OVFd8fVkY10qH7QMi5pRJvyZx9R9bih36s,217
+llm_benchmark/data/benchmark_models_2gb_ram.yml,sha256=eQHxZfXnHSgfgPSxj6yKdPSiEbE-ayMGeJTTXSunqvs,101
+llm_benchmark/data/benchmark_models_3gb_ram.yml,sha256=7n3BTVdTIvZOvFZwfo_FUB_UtexD5XoPg6NpX5YZoLc,121
+llm_benchmark/data/benchmark_models_4gb_ram.yml,sha256=E85cugjAbeVXAnvKhe77VtZJ69x-guKOEUP2l3q5fLk,142
+llm_benchmark/data/benchmark_models_8gb_ram.yml,sha256=Ot3xfeezL5K65V0GC8ha9HOvtF8qSUk1A-71W4eQpo8,196
+llm_benchmark/data/img/sample1.jpg,sha256=q7ABTbWXdWrVqxmMiDjRmvKaXa8Ewckm4T3jmby09Vs,596759
+llm_benchmark/data/img/sample2.jpg,sha256=wYKHxYbsIScnP07_4X_iC_w0reIU-HNeSUw239yf-P0,243536
+llm_benchmark/data/img/sample3.jpg,sha256=UyFxJvw1DvZK-wzFVyu0tRQ-cX8MJnOz79KDyQSfn3E,518611
+llm_benchmark/data/img/sample4.jpg,sha256=t3VIGACJfMuHRPjdJEUKiwWquZSYnFYN09OXeCopO8I,236523
+llm_benchmark/data/img/sample5.jpg,sha256=rEAvbm2qrAJ2-UxurdHGfeStjx2KsNkJj_TEKAOfyTs,534025
+llm_benchmark/security_connection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+llm_benchmark/security_connection/connection.py,sha256=KbAXzsoKmDsMO_gIJgIWFLgm0xP23dirpWSg2Yb_2zw,723
+llm_benchmark/security_connection/security.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+llm_benchmark/systeminfo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+llm_benchmark/systeminfo/sysmain.py,sha256=AfIyGO3-S8JfFe2F6hQBD5CtJjXv_lyBIcjH_mI4PiM,8715
+llm_benchmark-0.3.27.dist-info/LICENSE,sha256=k_4utRVOlpXU23bis0RwhxbUOH84fxg37__WQ_pxzVY,1083
+llm_benchmark-0.3.27.dist-info/METADATA,sha256=5U73M2I3shy6IUJYhqNzmPAn6LASkbk5I5pCk3J45Gk,3449
+llm_benchmark-0.3.27.dist-info/WHEEL,sha256=9Hm2OB-j1QcCUq9Jguht7ayGIIZBRTdOXD1qg9cCgPM,109
+llm_benchmark-0.3.27.dist-info/entry_points.txt,sha256=JMAtMpNNt4cq0iB0z35kOP6tz-i3HmvtP9X6CtLHhyw,57
+llm_benchmark-0.3.27.dist-info/top_level.txt,sha256=SQ0wiRPk2zxai1IPFl5pDctlKZ19OJKZUKF1kJpbdHA,14
+llm_benchmark-0.3.27.dist-info/RECORD,,

llm_benchmark-0.3.27.dist-info/WHEEL

@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.8.0)
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any
+

llm_benchmark-0.3.27.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+llm_benchmark = llm_benchmark.main:app

llm_benchmark-0.3.27.dist-info/top_level.txt

@@ -0,0 +1 @@
+llm_benchmark