fr-cli 2.1.0__py3-none-any.whl

fr_cli/agent/builtins/rag_watcher_daemon.py

@@ -0,0 +1,156 @@
+"""
+RAG 知识库守护进程 —— 后台藏经阁守护
+负责在主进程退出后继续监控知识库目录，新文件自动向量化入库。
+
+启动方式（不应由用户直接调用）：
+    python -m fr_cli.agent.builtins.rag_watcher_daemon --kb_dir <目录> [--db_path <路径>] [--interval <秒>]
+
+停止方式：
+    创建 ~/.fr_cli_rag_watcher.stop 标记文件，守护进程检测到后自行退出。
+"""
+import os
+import sys
+import time
+import signal
+import atexit
+import argparse
+from pathlib import Path
+from datetime import datetime
+
+# 确保项目根目录在 Python 路径中
+_project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+if _project_root not in sys.path:
+    sys.path.insert(0, _project_root)
+
+PID_FILE = Path.home() / ".fr_cli_rag_watcher.pid"
+STOP_FILE = Path.home() / ".fr_cli_rag_watcher.stop"
+LOG_FILE = Path.home() / ".fr_cli_rag_watcher.log"
+DB_PATH = Path.home() / ".fr_cli_rag_db"
+
+DEFAULT_INTERVAL = 30
+
+
+def _log(msg):
+    """写入日志文件并打印到 stderr（便于调试）"""
+    ts = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    line = f"[{ts}] {msg}"
+    try:
+        with open(LOG_FILE, "a", encoding="utf-8") as f:
+            f.write(line + "\n")
+    except Exception:
+        pass
+    sys.stderr.write(line + "\n")
+    sys.stderr.flush()
+
+
+def _write_pid(pid):
+    try:
+        with open(PID_FILE, "w", encoding="utf-8") as f:
+            f.write(str(pid))
+    except Exception as e:
+        _log(f"写入 PID 文件失败: {e}")
+
+
+def _clear_stop_marker():
+    if STOP_FILE.exists():
+        try:
+            STOP_FILE.unlink()
+        except Exception:
+            pass
+
+
+def _cleanup():
+    _clear_stop_marker()
+    if PID_FILE.exists():
+        try:
+            PID_FILE.unlink()
+        except Exception:
+            pass
+    _log("守护进程已清理并退出。")
+
+
+def _setup_signal_handlers():
+    def _sigterm_handler(signum, frame):
+        _log(f"收到信号 {signum}，准备退出...")
+        _cleanup()
+        sys.exit(0)
+
+    signal.signal(signal.SIGTERM, _sigterm_handler)
+    if hasattr(signal, "SIGINT"):
+        signal.signal(signal.SIGINT, _sigterm_handler)
+
+
+def _parse_args():
+    parser = argparse.ArgumentParser(description="RAG Knowledge Base Watcher Daemon")
+    parser.add_argument("--kb_dir", required=True, help="知识库目录路径")
+    parser.add_argument("--db_path", default=str(DB_PATH), help="ChromaDB 持久化路径")
+    parser.add_argument("--interval", type=int, default=DEFAULT_INTERVAL, help="扫描间隔（秒，默认30）")
+    return parser.parse_args()
+
+
+def run_daemon():
+    """守护进程主循环"""
+    args = _parse_args()
+    kb_dir = Path(args.kb_dir)
+    db_path = Path(args.db_path)
+    interval = max(5, args.interval)
+
+    _clear_stop_marker()
+    _write_pid(os.getpid())
+    atexit.register(_cleanup)
+    _setup_signal_handlers()
+
+    _log("=" * 50)
+    _log(f"RAG 知识库守护进程启动")
+    _log(f"  知识库目录: {kb_dir}")
+    _log(f"  向量数据库: {db_path}")
+    _log(f"  扫描间隔:   {interval} 秒")
+    _log("=" * 50)
+
+    if not kb_dir.exists():
+        _log(f"错误: 知识库目录不存在: {kb_dir}")
+        _cleanup()
+        sys.exit(1)
+
+    # 延迟导入，减少启动开销
+    try:
+        from fr_cli.agent.builtins.rag import RAGManager
+    except Exception as e:
+        _log(f"导入 RAGManager 失败: {e}")
+        _cleanup()
+        sys.exit(1)
+
+    mgr = RAGManager(kb_dir=str(kb_dir), db_path=str(db_path))
+
+    # 首次全量同步
+    _log("开始首次全量同步...")
+    try:
+        ok, msg = mgr.sync_directory()
+        _log(f"首次同步结果: {msg}")
+    except Exception as e:
+        _log(f"首次同步异常: {e}")
+
+    # 主循环：定期检查停止标记并同步目录
+    scan_count = 0
+    while True:
+        time.sleep(interval)
+
+        if STOP_FILE.exists():
+            _log("检测到停止标记，准备退出...")
+            break
+
+        scan_count += 1
+        try:
+            ok, msg = mgr.sync_directory()
+            if ok and "已是最新状态" not in msg:
+                _log(f"扫描 #{scan_count}: {msg}")
+            elif scan_count % 120 == 0:  # 每约1小时（120次×30秒）记录一次心跳
+                _log(f"心跳 #{scan_count}: 知识库监控正常，{msg}")
+        except Exception as e:
+            _log(f"扫描 #{scan_count} 异常: {e}")
+
+    _cleanup()
+
+
+if __name__ == "__main__":
+    run_daemon()

fr_cli/agent/builtins/remote.py

@@ -0,0 +1,214 @@
+"""
+@remote 内置 Agent —— 远程 SSH 操作助手
+支持多机配置、配置向导、AI 生成远程命令。
+"""
+from pathlib import Path
+
+REMOTE_CFG_PATH = Path.home() / ".fr_cli_remotes.json"
+
+REMOTE_SYS_PROMPT = """你是一个远程系统命令专家。请根据目标主机的操作系统类型和用户需求，生成最合适的远程命令。
+
+规则：
+1. 只输出命令本身，不要任何解释、不要 markdown 代码块、不要多余文字
+2. 如果需求涉及危险操作，输出 COMMENT: 开头的注释警告
+3. 优先使用标准 Linux/Unix 命令（目标多为服务器）
+4. 如果需要多条命令，用 && 或 ; 连接成一行
+5. 避免交互式命令（如 vim、top），使用非交互式替代
+
+目标主机 OS: {os_name}
+"""
+
+
+def _load_hosts():
+    from fr_cli.agent.builtins._utils import load_json_config
+    return load_json_config(REMOTE_CFG_PATH)
+
+
+def _save_hosts(hosts):
+    from fr_cli.agent.builtins._utils import save_json_config
+    save_json_config(REMOTE_CFG_PATH, hosts)
+
+
+def list_hosts():
+    return _load_hosts()
+
+
+def save_host(alias, ip, port, user, auth_type, auth_value):
+    hosts = _load_hosts()
+    hosts[alias] = {
+        "ip": ip,
+        "port": int(port) if port else 22,
+        "user": user,
+        "auth_type": auth_type,  # "password" or "key"
+        "auth_value": auth_value,
+    }
+    _save_hosts(hosts)
+
+
+def delete_host(alias):
+    hosts = _load_hosts()
+    if alias in hosts:
+        del hosts[alias]
+        _save_hosts(hosts)
+        return True
+    return False
+
+
+def _exec_ssh(host_cfg, command):
+    """通过 ssh 命令执行远程操作（使用 paramiko 避免命令注入）"""
+    ip = host_cfg["ip"]
+    port = host_cfg.get("port", 22)
+    user = host_cfg["user"]
+    auth_type = host_cfg.get("auth_type", "password")
+    auth_value = host_cfg.get("auth_value", "")
+
+    try:
+        import paramiko
+    except ImportError:
+        return None, "缺少 paramiko (pip install paramiko)"
+
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    try:
+        connect_kwargs = {
+            "hostname": ip,
+            "port": int(port),
+            "username": user,
+            "timeout": 30,
+            "look_for_keys": False,
+        }
+        if auth_type == "key":
+            connect_kwargs["key_filename"] = auth_value
+        else:
+            connect_kwargs["password"] = auth_value
+
+        client.connect(**connect_kwargs)
+        stdin, stdout, stderr = client.exec_command(command)
+        out = stdout.read().decode("utf-8", errors="ignore")
+        err = stderr.read().decode("utf-8", errors="ignore")
+        client.close()
+        return out + err, None
+    except Exception as e:
+        return None, str(e)
+
+
+def _detect_os(host_cfg):
+    """探测远程主机操作系统"""
+    out, err = _exec_ssh(host_cfg, "uname -s")
+    if err:
+        return "Unknown", err
+    return out.strip() or "Linux", None
+
+
+def handle_remote(user_input, state):
+    """处理 @remote 前缀的请求"""
+    from fr_cli.core.stream import stream_cnt
+    from fr_cli.ui.ui import CYAN, GREEN, RED, YELLOW, DIM, RESET
+
+    hosts = list_hosts()
+    if not hosts:
+        print(f"{YELLOW}未配置远程主机。正在启动配置向导...{RESET}")
+        _setup_wizard(state.lang)
+        hosts = list_hosts()
+        if not hosts:
+            print(f"{RED}配置取消，无法执行远程操作。{RESET}")
+            return
+
+    # 解析输入: @remote [ip/alias] 需求
+    text = user_input[len("@remote"):].strip()
+    parts = text.split(None, 1)
+
+    # 如果只配置了一台，默认使用它
+    if len(hosts) == 1:
+        alias = list(hosts.keys())[0]
+        requirement = text
+    else:
+        if len(parts) < 2:
+            print(f"{YELLOW}用法: @remote <别名/IP> <需求描述>{RESET}")
+            print(f"{DIM}已配置主机: {', '.join(hosts.keys())}{RESET}")
+            return
+        alias = parts[0]
+        requirement = parts[1]
+
+    # 查找主机配置
+    host_cfg = hosts.get(alias)
+    if not host_cfg:
+        # 尝试用 alias 模糊匹配
+        for k, v in hosts.items():
+            if k.lower() == alias.lower() or v.get("ip") == alias:
+                host_cfg = v
+                alias = k
+                break
+    if not host_cfg:
+        print(f"{RED}未找到主机 [{alias}]。已配置: {', '.join(hosts.keys())}{RESET}")
+        return
+
+    # 探测 OS
+    os_name, err = _detect_os(host_cfg)
+    if err:
+        print(f"{RED}无法连接主机 [{alias}]: {err}{RESET}")
+        return
+
+    prompt = REMOTE_SYS_PROMPT.format(os_name=os_name)
+    messages = [
+        {"role": "system", "content": prompt},
+        {"role": "user", "content": requirement},
+    ]
+
+    print(f"{CYAN}🧙 正在为 [{alias}]({os_name}) 生成远程命令...{RESET}")
+    cmd_text, _, _ = stream_cnt(state.client, state.model_name, messages, state.lang, custom_prefix="", max_tokens=1024)
+    cmd_text = cmd_text.strip()
+
+    from fr_cli.agent.builtins._utils import strip_code_blocks
+    cmd_text = strip_code_blocks(cmd_text)
+
+    if not cmd_text:
+        print(f"{RED}未能生成有效命令。{RESET}")
+        return
+
+    if cmd_text.startswith("COMMENT:"):
+        print(f"{YELLOW}{cmd_text}{RESET}")
+        return
+
+    print(f"\n{DIM}建议命令 ({alias}):{RESET}\n{CYAN}{cmd_text}{RESET}")
+    from fr_cli.agent.builtins._utils import confirm_execute
+    if not confirm_execute():
+        print(f"{DIM}已取消。{RESET}")
+        return
+
+    out, err = _exec_ssh(host_cfg, cmd_text)
+    if err:
+        print(f"{RED}❌ 执行失败: {err}{RESET}")
+    else:
+        if out.strip():
+            print(f"\n{GREEN}{out.strip()[:3000]}{RESET}")
+        else:
+            print(f"{GREEN}✅ 命令执行完成（无输出）{RESET}")
+
+
+def _setup_wizard(lang="zh"):
+    """远程主机配置向导"""
+    from fr_cli.ui.ui import CYAN, GREEN, YELLOW, DIM, RESET
+
+    print(f"{CYAN}═══ 远程主机配置向导 ═══{RESET}")
+    alias = input(f"{DIM}别名 (如: myserver): {RESET}").strip()
+    if not alias:
+        print(f"{YELLOW}别名不能为空。{RESET}")
+        return
+    ip = input(f"{DIM}IP 地址: {RESET}").strip()
+    if not ip:
+        print(f"{YELLOW}IP 不能为空。{RESET}")
+        return
+    port = input(f"{DIM}端口 [22]: {RESET}").strip() or "22"
+    user = input(f"{DIM}用户名: {RESET}").strip()
+    if not user:
+        print(f"{YELLOW}用户名不能为空。{RESET}")
+        return
+    auth = input(f"{DIM}认证方式 (password/key) [password]: {RESET}").strip() or "password"
+    if auth == "key":
+        auth_value = input(f"{DIM}私钥文件路径 (如: ~/.ssh/id_rsa): {RESET}").strip()
+    else:
+        auth_value = input(f"{DIM}密码: {RESET}").strip()
+
+    save_host(alias, ip, port, user, auth, auth_value)
+    print(f"{GREEN}✅ 主机 [{alias}] ({ip}) 已保存。{RESET}")

fr_cli/agent/builtins/spider.py

@@ -0,0 +1,247 @@
+"""
+@spider 内置 Agent —— 智能网页爬虫助手
+模拟真人浏览行为，支持反爬自适应，使用 requests → selenium 降级策略。
+"""
+import random
+import re
+import time
+from datetime import datetime
+from pathlib import Path
+from urllib.parse import urljoin, urlparse
+
+# 尝试导入可选依赖
+_requests = None
+_selenium = None
+
+
+def _get_requests():
+    global _requests
+    if _requests is None:
+        try:
+            import requests as r
+            _requests = r
+        except ImportError:
+            pass
+    return _requests
+
+
+def _get_selenium():
+    global _selenium
+    if _selenium is None:
+        try:
+            from selenium import webdriver
+            from selenium.webdriver.common.by import By
+            from selenium.webdriver.common.action_chains import ActionChains
+            _selenium = {"webdriver": webdriver, "By": By, "ActionChains": ActionChains}
+        except ImportError:
+            pass
+    return _selenium
+
+
+USER_AGENTS = [
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
+]
+
+
+def _sanitize_filename(url):
+    """将 URL 转换为安全的文件名"""
+    parsed = urlparse(url)
+    path = parsed.path.strip("/").replace("/", "_")
+    if not path:
+        path = "index"
+    name = f"{parsed.netloc}_{path}"
+    name = re.sub(r'[^\w\-_]', '_', name)[:100]
+    return name + ".html"
+
+
+def _fetch_with_requests(url):
+    """使用 requests 获取页面"""
+    requests = _get_requests()
+    if not requests:
+        return None, "requests 未安装 (pip install requests)"
+    headers = {"User-Agent": random.choice(USER_AGENTS)}
+    try:
+        resp = requests.get(url, headers=headers, timeout=15)
+        resp.raise_for_status()
+        text = resp.text
+        # 简单反爬检测
+        if len(text) < 500 or "captcha" in text.lower() or "access denied" in text.lower():
+            return None, "可能触发反爬机制"
+        return text, None
+    except Exception as e:
+        return None, str(e)
+
+
+def _fetch_with_selenium(url):
+    """使用 selenium 模拟真人浏览"""
+    sel = _get_selenium()
+    if not sel:
+        return None, "selenium 未安装 (pip install selenium)"
+
+    webdriver = sel["webdriver"]
+    By = sel["By"]
+    ActionChains = sel["ActionChains"]
+
+    options = webdriver.ChromeOptions()
+    options.add_argument("--headless")
+    options.add_argument("--no-sandbox")
+    options.add_argument("--disable-dev-shm-usage")
+    options.add_argument(f"--user-agent={random.choice(USER_AGENTS)}")
+    options.add_argument("--window-size=1920,1080")
+
+    driver = None
+    try:
+        driver = webdriver.Chrome(options=options)
+        driver.get(url)
+
+        # 模拟真人行为
+        time.sleep(random.uniform(1.5, 3.0))
+
+        # 随机滚动
+        for _ in range(random.randint(2, 5)):
+            scroll_y = random.randint(100, 800)
+            driver.execute_script(f"window.scrollBy(0, {scroll_y});")
+            time.sleep(random.uniform(0.5, 1.5))
+
+        # 随机悬停
+        try:
+            elements = driver.find_elements(By.TAG_NAME, "a")
+            if elements:
+                el = random.choice(elements[:10])
+                ActionChains(driver).move_to_element(el).pause(random.uniform(0.3, 0.8)).perform()
+        except Exception:
+            pass
+
+        # 再滚动回顶部
+        driver.execute_script("window.scrollTo(0, 0);")
+        time.sleep(random.uniform(0.5, 1.0))
+
+        html = driver.page_source
+        return html, None
+    except Exception as e:
+        return None, str(e)
+    finally:
+        if driver:
+            driver.quit()
+
+
+def _extract_links(html, base_url):
+    """从 HTML 中提取同域名链接"""
+    links = set()
+    base_domain = urlparse(base_url).netloc
+    pattern = re.compile(r'href=["\']([^"\']+)["\']', re.I)
+    for m in pattern.finditer(html):
+        href = m.group(1)
+        full = urljoin(base_url, href)
+        if urlparse(full).netloc == base_domain:
+            # 去重和过滤
+            if not full.startswith("javascript:") and not full.startswith("mailto:"):
+                links.add(full)
+    return list(links)
+
+
+def _save_page(url, html, output_dir):
+    """保存页面到工作区"""
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    filename = _sanitize_filename(url)
+    filepath = output_dir / filename
+    filepath.write_text(html, encoding="utf-8")
+    return str(filepath)
+
+
+def crawl(url, depth=1, output_base=None, lang="zh"):
+    """爬取指定 URL，返回 (saved_files, errors)"""
+    from fr_cli.ui.ui import CYAN, GREEN, RED, DIM, YELLOW, RESET
+
+    if depth < 1:
+        depth = 1
+    if depth > 3:
+        depth = 3  # 限制最大深度为 3
+
+    if output_base is None:
+        output_base = Path.cwd() / f"web_{datetime.now().strftime('%Y%m%d')}"
+    else:
+        output_base = Path(output_base)
+
+    saved = []
+    errors = []
+    visited = set()
+    to_crawl = [(url, 0)]  # (url, current_depth)
+
+    while to_crawl:
+        current_url, current_depth = to_crawl.pop(0)
+        if current_url in visited or current_depth >= depth:
+            continue
+        visited.add(current_url)
+
+        print(f"{CYAN}🕷️ 爬取 [{current_depth+1}/{depth}]: {current_url[:80]}...{RESET}")
+
+        # 第一优先级: requests
+        html, err = _fetch_with_requests(current_url)
+        if err or not html:
+            print(f"{YELLOW}  requests 失败: {err or '内容为空'}，尝试 selenium...{RESET}")
+            html, err2 = _fetch_with_selenium(current_url)
+            if err2 or not html:
+                errors.append(f"{current_url}: {err2 or err}")
+                print(f"{RED}  ❌ 爬取失败: {err2 or err}{RESET}")
+                continue
+
+        # 保存页面
+        filepath = _save_page(current_url, html, output_base)
+        saved.append(filepath)
+        print(f"{GREEN}  ✅ 已保存: {filepath}{RESET}")
+
+        # 如果还有深度，提取链接继续爬
+        if current_depth + 1 < depth:
+            links = _extract_links(html, current_url)
+            for link in links[:20]:  # 每页最多20个链接
+                if link not in visited:
+                    to_crawl.append((link, current_depth + 1))
+            if links:
+                print(f"{DIM}  发现 {len(links)} 个链接，加入 {min(len(links), 20)} 个待爬取{RESET}")
+
+        # 礼貌延迟
+        time.sleep(random.uniform(1.0, 2.5))
+
+    return saved, errors
+
+
+def handle_spider(user_input, state):
+    """处理 @spider 前缀的请求"""
+    from fr_cli.ui.ui import CYAN, GREEN, RED, YELLOW, DIM, RESET
+
+    text = user_input[len("@spider"):].strip()
+    if not text:
+        print(f"{RED}用法: @spider <URL> [深度]{RESET}")
+        return
+
+    parts = text.split()
+    url = parts[0]
+    depth = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else 1
+
+    if not url.startswith("http"):
+        url = "https://" + url
+
+    # 检查依赖
+    if not _get_requests():
+        print(f"{RED}缺少依赖: pip install requests selenium{RESET}")
+        return
+
+    output_dir = None
+    if state.vfs and state.vfs.cwd:
+        output_dir = state.vfs.cwd
+
+    print(f"{CYAN}🕷️ 开始爬取: {url} | 深度: {depth}{RESET}")
+    saved, errors = crawl(url, depth, output_dir, state.lang)
+
+    print(f"\n{GREEN}═══ 爬取完成 ═══{RESET}")
+    print(f"{GREEN}  成功: {len(saved)} 个页面{RESET}")
+    if errors:
+        print(f"{RED}  失败: {len(errors)} 个页面{RESET}")
+        for e in errors[:5]:
+            print(f"{RED}    - {e}{RESET}")
+    if saved:
+        print(f"{DIM}  保存目录: {Path(saved[0]).parent}{RESET}")