ftllm 0.0.0.8__py3-none-win_amd64.whl

ftllm/__init__.py

@@ -0,0 +1,4 @@
+__all__ = ["llm"]
+
+from importlib.metadata import version
+__version__ = version("ftllm")  # 从安装的元数据读取

ftllm/chat.py

@@ -0,0 +1,56 @@
+import argparse
+from .util import make_normal_parser
+import readline
+
+def args_parser():
+    parser = make_normal_parser('fastllm_chat')
+    args = parser.parse_args()
+    return args
+
+def fastllm_chat(args):
+    from .util import make_normal_llm_model
+    model = make_normal_llm_model(args)
+
+    generation_config = {
+        'repetition_penalty': 1.0,
+        'top_p': 0.8,
+        'top_k': 1,
+        'temperature': 1.0
+    }
+    import os
+    import json
+    if (os.path.exists(os.path.join(args.path, "generation_config.json"))):
+        with open(os.path.join(args.path, "generation_config.json"), "r", encoding="utf-8") as file:
+            config = json.load(file)
+            if ('do_sample' in config and config['do_sample']):
+                for it in ["repetition_penalty", "top_p", "top_k", "temperature"]:
+                    if (it in config):
+                        generation_config[it] = config[it];
+
+    hint = "输入内容开始对话\n'clear'清空记录\n'stop'终止程序."
+    history = []
+
+    print(hint)
+    while True:
+        query = input("\nUser：")
+        if query.strip() == "stop":
+            break
+        if query.strip() == "clear":
+            history = []
+            print(hint)
+            continue
+        print("AI:", end = "");
+        curResponse = "";
+        for response in model.stream_response(query, history = history, 
+                                              repeat_penalty = generation_config["repetition_penalty"],
+                                              top_p = generation_config["top_p"],
+                                              top_k = generation_config["top_k"],
+                                              temperature = generation_config["temperature"]):
+            curResponse += response;
+            print(response, flush = True, end = "")
+        history.append((query, curResponse))
+    model.release_memory()
+
+if __name__ == "__main__":
+    args = args_parser()
+    fastllm_chat(args)

ftllm/cli.py

@@ -0,0 +1,141 @@
+import argparse
+from .util import make_normal_parser
+import readline
+import os
+
+def save_defaults_to_json(parser, filename):
+    # 获取所有参数的默认值
+    defaults = {}
+
+    # 添加位置参数的默认值
+    for action in parser._actions:
+        if not action.option_strings and action.dest != 'help':
+            defaults[action.dest] = action.default if action.default is not None else ""
+    if defaults["model"] == '':
+        defaults["model"] = "Qwen/Qwen2-0.5B-Instruct"
+    for action in parser._actions:
+        # 跳过位置参数（没有option_strings的）
+        if not action.option_strings:
+            continue
+            
+        # 获取参数名（选择最长的选项名，去掉前面的--或-）
+        name = max(action.option_strings, key=len).lstrip('-')
+        if (name == 'low' or name == 'path'):
+            continue
+        if (name == 'device'):
+            action.default = ""
+        # 处理store_true类型的参数
+        if action.const is True:
+            defaults[name] = False  # store_true参数的默认值是False
+        else:
+            defaults[name] = action.default
+    defaults["FASTLLM_USE_NUMA"] = "OFF"
+    defaults["FASTLLM_NUMA_THREADS"] = 27
+
+    # 将字典转换为JSON并保存到文件
+    import json
+    with open(filename, 'w', encoding='utf-8') as f:
+        json.dump(defaults, f, indent = 4, ensure_ascii = False)
+    
+    print("Create config to -> \"" + filename + "\"")
+
+def args_parser():
+    parser = argparse.ArgumentParser(description = "fastllm")
+    subparsers = parser.add_subparsers(dest='command', help='子命令')
+
+    # 创建共享的解析器
+    shared_parser = make_normal_parser("fastllm", add_help = False)
+
+    # 下载解析器
+    from ftllm.download import make_download_parser
+    download_parser = make_download_parser(add_help = False)
+
+    # 打开ui界面
+    ui_parser_ = subparsers.add_parser('ui', parents = [shared_parser], help = 'ui模式')
+
+    # 创建chat子命令（使用共享解析器）
+    chat_parser_ = subparsers.add_parser('chat', parents = [shared_parser], help = '聊天模式')
+
+    # 创建run子命令（使用相同的共享解析器）
+    run_parser_ = subparsers.add_parser('run', parents = [shared_parser], help = '运行模式')
+
+    download_parser_ = subparsers.add_parser('download', parents = [download_parser], help = '下载模型')
+
+    # 创建webui子命令（独立的解析器）
+    webui_parser_ = subparsers.add_parser('webui', parents = [shared_parser], help='Web UI')
+    webui_parser_.add_argument('--port', type = int, default = 1616, help = '端口号')
+
+    server_parser = shared_parser
+    from ftllm.util import add_server_args
+    add_server_args(server_parser)
+    server_parser_ = subparsers.add_parser('serve', parents = [server_parser], help = 'api模式')
+    serve_parser_ = subparsers.add_parser('server', parents = [server_parser], help = 'api模式')
+
+    config_parser_ = subparsers.add_parser('config', help = '创建配置文件')
+    config_parser_.add_argument('file', nargs='?', help = '配置文件的路径')
+
+    parser.add_argument('-v', '--version', action='store_true', help='输出版本号并退出')
+
+    return parser
+
+def main():
+    args = args_parser().parse_args()
+    if (args.version):
+        from . import __version__
+        print("ftllm version: " + __version__)
+        return
+    # 根据不同的子命令执行不同的操作
+    if args.command == 'ui':
+        from .ui import FastllmStartUI
+        FastllmStartUI()
+    if args.command == 'config':
+        file = args.file
+        if not(file) or file == '':
+            file = "config.json"
+        if os.path.exists(file):
+            choice = input("File '" + file + "' exists，replace it? (Y/N): ").strip().upper()
+            if choice == 'Y':
+                pass
+            else:
+                return
+        save_defaults_to_json(make_normal_parser("fastllm", add_help = False), file)
+    elif args.command in ('chat', 'run'):
+        from ftllm.chat import fastllm_chat
+        fastllm_chat(args)
+    elif args.command == "download":
+        from ftllm.download import HFDDownloader
+        HFDDownloader(args).run()
+    elif args.command == 'webui':
+        current_path = os.path.dirname(os.path.abspath(__file__))
+        web_demo_path = os.path.join(current_path, 'web_demo.py')
+        
+        args_dict = vars(args)
+        args_dict.pop('command', None)
+        port = args_dict.pop('port', 1616)
+        model = args_dict.pop('model', '')
+        # Convert remaining arguments to command line format
+        args_list = []
+        if (model != ''):
+            args_list.append(model)
+        for key, value in args_dict.items():
+            if value is not None:
+                if isinstance(value, bool):
+                    if value:
+                        args_list.append(f"--{key}")
+                elif value != '':
+                    args_list.append(f"--{key} {value}")
+        
+        # Build the command
+        cmd = f"streamlit run --server.port {port} {web_demo_path} -- {' '.join(args_list)}"
+        print(f"Running: {cmd}")
+        os.system(cmd)
+        return
+    elif args.command in ('server', 'serve'):
+        from ftllm.server import fastllm_server
+        fastllm_server(args)
+    else:
+        print("Invalid command: ", args.command)
+        exit(0)
+
+if __name__ == "__main__":
+    main()

ftllm/download.py

@@ -0,0 +1,282 @@
+#!/usr/bin/env python3
+import os
+import re
+import json
+import argparse
+import subprocess
+import sys
+from pathlib import Path
+from fnmatch import translate
+from typing import List, Tuple
+import requests
+from .util import get_fastllm_cache_path
+from .util import make_download_parser
+
+def find_metadata(repo_id) -> bool:
+    hf_endpoint = os.environ.get("HF_ENDPOINT", "https://hf-mirror.com")
+    url = f"{hf_endpoint}/api/models/{repo_id}"
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        metadata = response.json()
+        return True
+    except requests.HTTPError as e:
+        return False
+
+def search_model(repo_id):
+    hf_endpoint = os.environ.get("HF_ENDPOINT", "https://hf-mirror.com")
+    url = f"{hf_endpoint}/api/models?search={repo_id}&limit=3"
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        return response.json()
+    except requests.HTTPError as e:
+        return []
+# ANSI颜色代码
+COLORS = {
+    "RED": "\033[0;31m",
+    "GREEN": "\033[0;32m",
+    "YELLOW": "\033[1;33m",
+    "NC": "\033[0m"
+}
+
+def color_print(text: str, color: str) -> None:
+    print(f"{COLORS[color]}{text}{COLORS['NC']}", flush=True)
+
+class HFDArgs:
+    def __init__(self):
+        self.repo_id = ""
+        self.include = []
+        self.exclude = []
+        self.hf_username = ""
+        self.hf_token = ""
+        self.tool = "aria2c"
+        self.x = 4
+        self.j = 5
+        self.dataset = False
+        self.local_dir = ""
+        self.revision = "main"
+
+class HFDDownloader:
+    def __init__(self, args):
+        self.args = args
+        self.validate_arguments()
+        self.setup_paths()
+        self.hf_endpoint = os.environ.get("HF_ENDPOINT", "https://hf-mirror.com")
+        
+        # 确保缓存目录存在
+        (self.local_dir / ".hfd").mkdir(parents=True, exist_ok=True)
+    
+    def validate_arguments(self):
+        for attr in ["x", "j"]:
+            value = getattr(self.args, attr)
+            if not (1 <= value <= 10):
+                color_print(f"{attr} must be between 1-10", "RED")
+                sys.exit(1)
+        
+    def setup_paths(self):
+        # 处理本地目录路径
+        if self.args.local_dir:
+            self.local_dir = Path(self.args.local_dir).expanduser().resolve()
+        else:
+            repo_name = self.args.repo_id.split("/")[-1] if "/" in self.args.repo_id else self.args.repo_id
+            self.local_dir = Path.cwd() / repo_name
+        
+        self.metadata_file = self.local_dir / ".hfd" / "repo_metadata.json"
+        self.command_file = self.local_dir / ".hfd" / "last_command"
+        self.fileslist_file = self.local_dir / ".hfd" / f"{self.args.tool}_urls.txt"
+
+    def run(self):
+        try:
+            finish_flag = os.path.join(self.local_dir, "FASTLLM_FINISH_FLAG")
+            if (os.path.exists(finish_flag)):
+                return
+            # 获取并处理元数据
+            metadata = self.fetch_metadata()
+            self.check_authentication(metadata)
+            
+            # 生成下载列表
+            if self.should_regenerate_filelist():
+                self.generate_filelist(metadata)
+            
+            # 执行下载
+            self.execute_download()
+            
+            color_print("Download completed successfully.", "GREEN")
+            with open(finish_flag, 'w') as file:
+                file.write('finish')
+        except KeyboardInterrupt:
+            color_print("\nDownload interrupted. You can resume by re-running the command.", "YELLOW")
+            sys.exit(1)
+        except Exception as e:
+            color_print(f"Error: {str(e)}", "RED")
+            sys.exit(1)
+
+    def fetch_metadata(self) -> dict:
+        color_print("Fetching repository metadata...", "YELLOW")
+        
+        api_path = "datasets" if self.args.dataset else "models"
+        url = f"{self.hf_endpoint}/api/{api_path}/{self.args.repo_id}"
+        if self.args.revision != "main":
+            url += f"/revision/{self.args.revision}"
+            
+        headers = {"Authorization": f"Bearer {self.args.hf_token}"} if self.args.hf_token else {}
+        
+        try:
+            response = requests.get(url, headers=headers)
+            response.raise_for_status()
+            metadata = response.json()
+            
+            # 保存元数据
+            with open(self.metadata_file, "w") as f:
+                json.dump(metadata, f)
+                
+            return metadata
+        except requests.HTTPError as e:
+            color_print(f"Failed to fetch metadata: {e}", "RED")
+            sys.exit(1)
+
+    def check_authentication(self, metadata: dict):
+        if metadata.get("gated", False) and not (self.args.hf_username and self.args.hf_token):
+            color_print("This repository requires authentication. Please provide --hf_username and --hf_token.", "RED")
+            sys.exit(1)
+
+    def should_regenerate_filelist(self) -> bool:
+        current_command = self.get_current_command()
+        
+        try:
+            with open(self.command_file) as f:
+                previous_command = f.read()
+            if current_command == previous_command and self.fileslist_file.exists():
+                color_print("Using cached file list.", "GREEN")
+                return False
+        except FileNotFoundError:
+            pass
+        
+        # 保存当前命令
+        with open(self.command_file, "w") as f:
+            f.write(current_command)
+        return True
+    
+    def get_current_command(self) -> str:
+        return " ".join(sys.argv[1:])
+
+    def generate_filelist(self, metadata: dict):
+        color_print("Generating download file list...", "YELLOW")
+        
+        include_regex = self.patterns_to_regex(self.args.include)
+        exclude_regex = self.patterns_to_regex(self.args.exclude)
+        
+        files = []
+        for item in metadata.get("siblings", []):
+            rfilename = item.get("rfilename")
+            if not rfilename:
+                continue
+            
+            # 过滤文件
+            if include_regex and not re.search(include_regex, rfilename):
+                continue
+            if exclude_regex and re.search(exclude_regex, rfilename):
+                continue
+            
+            files.append(rfilename)
+        
+        # 生成下载列表
+        with open(self.fileslist_file, "w") as f:
+            for file in files:
+                url = f"{self.hf_endpoint}/{self.args.repo_id}/resolve/{self.args.revision}/{file}"
+                if self.args.tool == "aria2c":
+                    lines = [
+                        url,
+                        f" dir={os.path.dirname(file)}",
+                        f" out={os.path.basename(file)}"
+                    ]
+                    if self.args.hf_token:
+                        lines.append(f" header=Authorization: Bearer {self.args.hf_token}")
+                    f.write("\n".join(lines) + "\n\n")
+                else:
+                    f.write(url + "\n")
+
+    @staticmethod
+    def patterns_to_regex(patterns: List[str]) -> str:
+        if not patterns:
+            return ""
+        regexes = [translate(p).replace(r"\Z(?ms)", r"$") for p in patterns]
+        return "|".join(regexes)
+
+    def execute_download(self):
+        color_print(f"Starting download with {self.args.tool}...", "YELLOW")
+        
+        os.chdir(self.local_dir)
+        if self.args.tool == "aria2c":
+            cmd = [
+                "aria2c",
+                "--console-log-level=error",
+                "--file-allocation=none",
+                "-x", str(self.args.x),
+                "-j", str(self.args.j),
+                "-s", str(self.args.x),
+                "-k", "1M",
+                "-c",
+                "-i", str(self.fileslist_file),
+                "--save-session", str(self.fileslist_file)
+            ]
+        else:
+            cmd = [
+                "wget",
+                "-x",
+                "-nH",
+                f"--cut-dirs={5 if self.args.dataset else 4}",
+                "--continue",
+                "--input-file", str(self.fileslist_file)
+            ]
+            if self.args.hf_token:
+                cmd.append(f"--header=Authorization: Bearer {self.args.hf_token}")
+        
+        try:
+            subprocess.run(cmd, check=True)
+        except subprocess.CalledProcessError as e:
+            color_print(f"Download failed: {e}", "RED")
+            sys.exit(1)
+
+class HFDNormalDownloader(HFDDownloader):
+    def __init__(self,
+                 repo_id: str,
+                 include = [],
+                 exclude = [],
+                 hf_username = "",
+                 hf_token = "",
+                 tool = "aria2c",
+                 x = 4,
+                 j = 5,
+                 dataset = False,
+                 local_dir = "",
+                 revision = "main"):
+        self.args = HFDArgs()
+        self.args.include = include
+        self.args.exclude = exclude
+        self.args.hf_username = hf_username
+        self.args.hf_token = hf_token
+        self.args.tool = tool
+        self.args.x = x
+        self.args.j = j
+        self.args.dataset = dataset
+        self.args.repo_id = repo_id
+        self.args.revision = revision
+        self.args.local_dir = local_dir
+        print("Model dir:", self.args.local_dir)
+        self.validate_arguments()
+        self.setup_paths()
+        self.hf_endpoint = os.environ.get("HF_ENDPOINT", "https://hf-mirror.com")
+        # 确保缓存目录存在
+        (self.local_dir / ".hfd").mkdir(parents=True, exist_ok=True)
+
+if __name__ == "__main__":
+    try:
+        downloader = HFDDownloader()
+        #model_name = "Qwen/Qwen2-1.5B-Instruct"
+        #downloader = HFDNormalDownloader(model_name, local_dir = get_fastllm_cache_path(model_name))
+        downloader.run()
+    except KeyboardInterrupt:
+        color_print("\nDownload interrupted.", "YELLOW")
+        sys.exit(1)

ftllm/export.py

@@ -0,0 +1,15 @@
+import argparse
+from ftllm import llm
+from .util import make_normal_parser
+from .util import make_normal_llm_model
+import readline
+
+def args_parser():
+    parser = make_normal_parser('fastllm_export')
+    parser.add_argument('-o', '--output', type = str, required = True, help = '导出路径')
+    args = parser.parse_args()
+    return args
+
+if __name__ == "__main__":
+    args = args_parser()
+    llm.export_llm_model_fromhf(path = args.path, dtype = args.dtype, lora = args.lora, output = args.output)