ai-dev-flow 3.10.0__py3-none-any.whl

ai_dev_flow/semantic_summarizer.py

@@ -0,0 +1,351 @@
+"""语义摘要器 — 从文件中提取结构化关键信息，替代纯截断。
+
+策略（零 AI 成本）：
+- 代码文件：提取类/函数签名、参数、返回类型、docstring、关键注释
+- 文档文件：提取标题层级 + 首句摘要
+- 配置文件：提取顶层结构
+- 提取结果缓存到 .ai-dev/summaries/{hash}.md
+"""
+
+from __future__ import annotations
+
+import re
+import hashlib
+import logging
+from pathlib import Path
+
+logger = logging.getLogger("ai-dev-flow")
+
+SUMMARY_DIR = "summaries"
+MAX_SUMMARY_LINES = 120  # 摘要最大行数
+
+
+class SemanticSummarizer:
+    """文件语义结构提取器。不调用 AI，基于语言规则提取关键信息。"""
+
+    def __init__(self, ai_dev_dir: Path):
+        self.cache_dir = ai_dev_dir / SUMMARY_DIR
+        self.cache_dir.mkdir(exist_ok=True)
+
+    def summarize(self, filepath: Path) -> str | None:
+        """对文件做语义摘要。已在缓存中则直接返回。"""
+        if not filepath.exists():
+            return None
+
+        try:
+            content = filepath.read_text(encoding="utf-8", errors="replace")
+        except Exception:
+            return None
+
+        cache_key = hashlib.sha256(content.encode()).hexdigest()[:16]
+        cache_file = self.cache_dir / f"{cache_key}.md"
+        if cache_file.exists():
+            return cache_file.read_text(encoding="utf-8")
+
+        ext = filepath.suffix.lower()
+        lines = content.splitlines()
+        size_kb = len(content.encode("utf-8")) // 1024
+
+        if ext in (".py", ".pyw"):
+            summary = self._summarize_python(lines, filepath, len(lines), size_kb)
+        elif ext in (".java", ".kt", ".scala"):
+            summary = self._summarize_java(lines, filepath, len(lines), size_kb)
+        elif ext in (".ts", ".tsx", ".js", ".jsx"):
+            summary = self._summarize_typescript(lines, filepath, len(lines), size_kb)
+        elif ext in (".md", ".mdx"):
+            summary = self._summarize_markdown(lines, filepath, len(lines), size_kb)
+        elif ext in (".yml", ".yaml"):
+            summary = self._summarize_yaml(lines, filepath, len(lines), size_kb)
+        elif ext in (".vue", ".svelte"):
+            summary = self._summarize_vue(lines, filepath, len(lines), size_kb)
+        elif ext in (".xml", ".html", ".htm"):
+            summary = self._summarize_xml(lines, filepath, len(lines), size_kb)
+        elif ext in (".sql"):
+            summary = self._summarize_sql(lines, filepath, len(lines), size_kb)
+        else:
+            summary = self._summarize_generic(lines, filepath, len(lines), size_kb)
+
+        cache_file.write_text(summary, encoding="utf-8")
+        return summary
+
+    # ---- Python ----
+
+    def _summarize_python(self, lines: list[str], filepath: Path,
+                          total: int, size_kb: int) -> str:
+        parts = [f"## File: {filepath.name} ({total} lines, {size_kb}KB)\n"]
+        imports, classes, funcs, constants, comments = [], [], [], [], []
+
+        for i, line in enumerate(lines, 1):
+            stripped = line.strip()
+            if not stripped:
+                continue
+
+            # Imports
+            if stripped.startswith(("import ", "from ")):
+                imports.append(stripped)
+            # Class defs
+            elif stripped.startswith("class "):
+                doc = self._peek_docstring(lines, i)
+                classes.append(f"- `{stripped}`{doc}")
+            # Function defs (not methods — those are indented)
+            elif stripped.startswith("def ") and not line.startswith((" ", "\t")):
+                doc = self._peek_docstring(lines, i)
+                funcs.append(f"- `{stripped}`{doc}")
+            # Decorators (capture before class/def)
+            elif stripped.startswith("@"):
+                if i < total and lines[i].strip().startswith(("class ", "def ")):
+                    continue  # will be captured with the class/def
+            # Module-level constants
+            elif re.match(r'^[A-Z_][A-Z0-9_]*\s*=', stripped):
+                constants.append(stripped.split("=")[0].strip())
+            # Key comments
+            elif re.search(r'(TODO|FIXME|HACK|XXX|NOTE|WARNING)', stripped, re.I):
+                comments.append(f"  L{i}: {stripped[:100]}")
+
+        if imports:
+            parts.append(f"**Imports**: {', '.join(imports[:15])}")
+            if len(imports) > 15:
+                parts.append(f"  ... and {len(imports) - 15} more")
+        if constants:
+            parts.append(f"**Constants**: {', '.join(constants[:20])}")
+        if classes:
+            parts.append(f"\n### Classes ({len(classes)})")
+            parts.extend(classes[:30])
+            if len(classes) > 30:
+                parts.append(f"  ... and {len(classes) - 30} more")
+        if funcs:
+            parts.append(f"\n### Functions ({len(funcs)})")
+            parts.extend(funcs[:30])
+            if len(funcs) > 30:
+                parts.append(f"  ... and {len(funcs) - 30} more")
+        if comments:
+            parts.append("\n### Key Comments")
+            parts.extend(comments[:15])
+
+        return "\n".join(parts)
+
+    # ---- Java/Kotlin ----
+
+    def _summarize_java(self, lines: list[str], filepath: Path,
+                        total: int, size_kb: int) -> str:
+        parts = [f"## File: {filepath.name} ({total} lines, {size_kb}KB)\n"]
+        package, imports, classes, methods, fields, comments = "", [], [], [], [], []
+
+        for i, line in enumerate(lines, 1):
+            stripped = line.strip()
+            if not stripped:
+                continue
+
+            if stripped.startswith("package "):
+                package = stripped
+            elif stripped.startswith("import "):
+                imports.append(stripped)
+            elif re.match(r'(public |private |protected )?(class |interface |@interface |enum )', stripped):
+                classes.append(f"- `{stripped}`")
+            elif re.match(r'(public |private |protected )?[\w<>\[\],\s]+\s+\w+\s*\(', stripped):
+                methods.append(f"- `{stripped[:120]}`")
+            elif re.match(r'(public |private |protected )?static\s+final\s+', stripped):
+                fields.append(stripped[:80])
+            elif re.search(r'(TODO|FIXME|HACK|XXX)', stripped, re.I):
+                comments.append(f"  L{i}: {stripped[:100]}")
+
+        if package:
+            parts.append(f"**Package**: {package}")
+        if imports:
+            parts.append(f"**Imports**: {len(imports)} packages")
+        if classes:
+            parts.append(f"\n### Classes/Interfaces ({len(classes)})")
+            parts.extend(classes)
+        if fields:
+            parts.append("\n### Constants/Fields")
+            parts.extend(fields[:15])
+        if methods:
+            parts.append(f"\n### Methods ({len(methods)})")
+            parts.extend(methods[:40])
+            if len(methods) > 40:
+                parts.append(f"  ... and {len(methods) - 40} more")
+        if comments:
+            parts.append("\n### Key Comments")
+            parts.extend(comments[:15])
+
+        return "\n".join(parts)
+
+    # ---- TypeScript/JavaScript ----
+
+    def _summarize_typescript(self, lines: list[str], filepath: Path,
+                              total: int, size_kb: int) -> str:
+        parts = [f"## File: {filepath.name} ({total} lines, {size_kb}KB)\n"]
+        imports, exports, classes, funcs, interfaces, comments = [], [], [], [], [], []
+
+        for i, line in enumerate(lines, 1):
+            stripped = line.strip()
+            if not stripped:
+                continue
+
+            if stripped.startswith("import "):
+                imports.append(stripped[:80])
+            elif stripped.startswith("export "):
+                exports.append(stripped[:100])
+            elif stripped.startswith("class "):
+                classes.append(f"- `{stripped[:120]}`")
+            elif re.match(r'(export\s+)?(interface|type)\s+', stripped):
+                interfaces.append(f"- `{stripped[:120]}`")
+            elif re.match(r'(export\s+)?(async\s+)?function\s+', stripped):
+                funcs.append(f"- `{stripped[:120]}`")
+            elif re.search(r'(TODO|FIXME|HACK|XXX)', stripped, re.I):
+                comments.append(f"  L{i}: {stripped[:100]}")
+
+        if imports:
+            parts.append(f"**Imports**: {len(imports)} modules")
+        if exports:
+            parts.append(f"**Exports**: {len(exports)} items")
+        if interfaces:
+            parts.append(f"\n### Interfaces/Types ({len(interfaces)})")
+            parts.extend(interfaces[:20])
+        if classes:
+            parts.append(f"\n### Classes ({len(classes)})")
+            parts.extend(classes[:20])
+        if funcs:
+            parts.append(f"\n### Functions ({len(funcs)})")
+            parts.extend(funcs[:30])
+        if comments:
+            parts.append("\n### Key Comments")
+            parts.extend(comments[:15])
+
+        return "\n".join(parts)
+
+    # ---- Markdown ----
+
+    def _summarize_markdown(self, lines: list[str], filepath: Path,
+                            total: int, size_kb: int) -> str:
+        parts = [f"## File: {filepath.name} ({total} lines, {size_kb}KB)\n"]
+        headers = []
+        for line in lines:
+            stripped = line.strip()
+            if stripped.startswith("#"):
+                headers.append(stripped)
+        if headers:
+            parts.append("### Document Structure")
+            parts.extend(headers[:40])
+            if len(headers) > 40:
+                parts.append(f"  ... and {len(headers) - 40} more headings")
+        return "\n".join(parts)
+
+    # ---- YAML ----
+
+    def _summarize_yaml(self, lines: list[str], filepath: Path,
+                        total: int, size_kb: int) -> str:
+        parts = [f"## File: {filepath.name} ({total} lines, {size_kb}KB)\n"]
+        top_keys = []
+        for line in lines:
+            stripped = line.strip()
+            if stripped and not stripped.startswith("#") and not line.startswith(" "):
+                top_keys.append(stripped[:100])
+        if top_keys:
+            parts.append("### Top-level Structure")
+            parts.extend(f"- {k}" for k in top_keys[:30])
+        return "\n".join(parts)
+
+    # ---- Vue/Svelte ----
+
+    def _summarize_vue(self, lines: list[str], filepath: Path,
+                       total: int, size_kb: int) -> str:
+        parts = [f"## File: {filepath.name} ({total} lines, {size_kb}KB)\n"]
+        sections = []
+        in_script, in_style, in_template = False, False, False
+        script_lines, style_lines, template_lines = 0, 0, 0
+
+        for line in lines:
+            stripped = line.strip()
+            if "<script" in stripped:
+                in_script = True
+            elif "</script>" in stripped:
+                in_script = False
+                sections.append(f"- `<script>` ({script_lines} lines)")
+            elif "<style" in stripped:
+                in_style = True
+            elif "</style>" in stripped:
+                in_style = False
+                sections.append(f"- `<style>` ({style_lines} lines)")
+            elif "<template" in stripped:
+                in_template = True
+            elif "</template>" in stripped:
+                in_template = False
+                sections.append(f"- `<template>` ({template_lines} lines)")
+            elif in_script:
+                script_lines += 1
+            elif in_style:
+                style_lines += 1
+            elif in_template:
+                template_lines += 1
+
+        if sections:
+            parts.append("### Component Structure")
+            parts.extend(sections)
+        return "\n".join(parts)
+
+    # ---- XML/HTML ----
+
+    def _summarize_xml(self, lines: list[str], filepath: Path,
+                       total: int, size_kb: int) -> str:
+        parts = [f"## File: {filepath.name} ({total} lines, {size_kb}KB)\n"]
+        tags = set()
+        for line in lines:
+            for m in re.finditer(r'<(\w+)', line):
+                tags.add(m.group(1))
+        if tags:
+            parts.append(f"**Elements**: {', '.join(sorted(tags)[:30])}")
+        return "\n".join(parts)
+
+    # ---- SQL ----
+
+    def _summarize_sql(self, lines: list[str], filepath: Path,
+                       total: int, size_kb: int) -> str:
+        parts = [f"## File: {filepath.name} ({total} lines, {size_kb}KB)\n"]
+        statements = []
+        for line in lines:
+            stripped = line.strip().upper()
+            if re.match(r'(CREATE|ALTER|DROP|INSERT|UPDATE|DELETE|SELECT|GRANT|REVOKE)\b', stripped):
+                statements.append(stripped[:120])
+        if statements:
+            parts.append("### SQL Statements")
+            parts.extend(f"- {s}" for s in statements[:20])
+            if len(statements) > 20:
+                parts.append(f"  ... and {len(statements) - 20} more")
+        return "\n".join(parts)
+
+    # ---- Generic fallback ----
+
+    def _summarize_generic(self, lines: list[str], filepath: Path,
+                           total: int, size_kb: int) -> str:
+        parts = [f"## File: {filepath.name} ({total} lines, {size_kb}KB)\n"]
+        parts.append(f"**Type**: {filepath.suffix or 'text'}")
+        # Extract lines that look like declarations
+        decls = []
+        for line in lines[:200]:
+            stripped = line.strip()
+            if stripped and not stripped.startswith(("#", "//", "/*", "*")):
+                if re.match(r'^[\w]+\s+[\w]+\s*[=(]', stripped):
+                    decls.append(stripped[:120])
+        if decls:
+            parts.append("### Possible Declarations")
+            parts.extend(f"- {d}" for d in decls[:30])
+        return "\n".join(parts)
+
+    # ---- Helpers ----
+
+    def _peek_docstring(self, lines: list[str], idx: int) -> str:
+        """提取紧随定义行的 docstring 首行。"""
+        if idx >= len(lines):
+            return ""
+        next_line = lines[idx].strip()
+        # 单行 docstring: """text""" 或 '''text'''
+        m = re.match(r'["\']{3}(.+?)["\']{3}', next_line)
+        if m:
+            return f' — "{m.group(1)[:80]}"'
+        # 多行 docstring 开始
+        if next_line.startswith(('"""', "'''")):
+            text = next_line.lstrip('"').lstrip("'").strip()
+            if text:
+                return f' — "{text[:80]}"'
+        return ""

ai_dev_flow/snapshot.py

@@ -0,0 +1,210 @@
+"""项目快照 — 文件树 hash + 变更检测，避免每次全量扫描项目。
+
+增量策略：
+- 首次运行：全量扫描，存 mtime+size 到 snapshot.json
+- 后续运行：对比 mtime/size，仅对变化文件重读内容
+- 内容缓存：未变化文件复用上次读取的内容
+"""
+
+from __future__ import annotations
+
+import json
+import hashlib
+import logging
+from pathlib import Path
+from dataclasses import dataclass
+
+logger = logging.getLogger("ai-dev-flow")
+
+SNAPSHOT_FILE = "snapshot.json"
+
+# 扫描时跳过的目录
+SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv", "venv",
+             "target", "build", "dist", ".idea", ".vscode", ".ai-dev"}
+
+# 二进制文件扩展名（不做内容缓存）
+BINARY_EXTS = {".jar", ".war", ".class", ".exe", ".dll", ".so", ".dylib",
+               ".png", ".jpg", ".jpeg", ".gif", ".ico", ".pdf", ".zip", ".gz"}
+
+
+@dataclass
+class FileEntry:
+    path: str          # 相对于 project_root 的路径
+    size: int
+    mtime: float
+    hash: str = ""     # 仅变更文件计算
+
+
+@dataclass
+class SnapshotDiff:
+    added: list[str]
+    removed: list[str]
+    modified: list[str]
+    unchanged: list[str]
+
+
+class SnapshotManager:
+    """管理项目文件树快照，支持增量变更检测。"""
+
+    def __init__(self, ai_dev_dir: Path, project_root: Path):
+        self.snapshot_path = ai_dev_dir / SNAPSHOT_FILE
+        self.project_root = project_root
+        self._content_cache: dict[str, str] = {}
+
+    # ---- 快照构建 ----
+
+    def build_snapshot(self) -> dict[str, FileEntry]:
+        """全量扫描项目，构建文件树快照。"""
+        entries: dict[str, FileEntry] = {}
+        self._walk(self.project_root, self.project_root, entries)
+        logger.info(f"Snapshot built: {len(entries)} files indexed")
+        return entries
+
+    def _walk(self, base: Path, current: Path, entries: dict[str, FileEntry]) -> None:
+        try:
+            for child in sorted(current.iterdir()):
+                if child.name.startswith(".") or child.name in SKIP_DIRS:
+                    continue
+                if child.is_dir():
+                    self._walk(base, child, entries)
+                elif child.is_file():
+                    try:
+                        stat = child.stat()
+                        rel = str(child.relative_to(base)).replace("\\", "/")
+                        entries[rel] = FileEntry(
+                            path=rel,
+                            size=stat.st_size,
+                            mtime=stat.st_mtime,
+                        )
+                    except OSError:
+                        pass
+        except PermissionError:
+            pass
+
+    # ---- 持久化 ----
+
+    def save(self, entries: dict[str, FileEntry]) -> None:
+        data = {
+            "project_root": str(self.project_root),
+            "files": {
+                path: {"size": e.size, "mtime": e.mtime, "hash": e.hash}
+                for path, e in entries.items()
+            }
+        }
+        self.snapshot_path.write_text(json.dumps(data, indent=2), encoding="utf-8")
+
+    def load(self) -> dict[str, FileEntry] | None:
+        if not self.snapshot_path.exists():
+            return None
+        try:
+            data = json.loads(self.snapshot_path.read_text(encoding="utf-8"))
+            entries = {}
+            for path, info in data.get("files", {}).items():
+                entries[path] = FileEntry(
+                    path=path,
+                    size=info["size"],
+                    mtime=info["mtime"],
+                    hash=info.get("hash", ""),
+                )
+            return entries
+        except (json.JSONDecodeError, KeyError) as e:
+            logger.warning(f"Snapshot corrupted: {e}")
+            return None
+
+    # ---- 变更检测 ----
+
+    def diff(self, before: dict[str, FileEntry],
+             after: dict[str, FileEntry]) -> SnapshotDiff:
+        """对比新旧快照，返回变更分类。"""
+        added = [p for p in after if p not in before]
+        removed = [p for p in before if p not in after]
+        modified = []
+        unchanged = []
+
+        for p in before:
+            if p in after:
+                bf = before[p]
+                af = after[p]
+                if bf.mtime != af.mtime or bf.size != af.size:
+                    modified.append(p)
+                else:
+                    unchanged.append(p)
+
+        return SnapshotDiff(
+            added=added, removed=removed, modified=modified, unchanged=unchanged
+        )
+
+    def get_changed_files(self) -> SnapshotDiff:
+        """主 API：对比当前文件系统与上次快照，返回变更文件列表。"""
+        old = self.load()
+        new = self.build_snapshot()
+        if old is None:
+            # 首次运行：全部文件视为 added
+            return SnapshotDiff(
+                added=list(new.keys()),
+                removed=[],
+                modified=[],
+                unchanged=[],
+            )
+        diff = self.diff(old, new)
+        # 对 modified 文件计算内容 hash 以确认真正变更
+        real_modified = []
+        for p in diff.modified:
+            if self._hash_changed(p, old[p], new[p]):
+                real_modified.append(p)
+            else:
+                diff.unchanged.append(p)
+        diff.modified = real_modified
+        return diff
+
+    def _hash_changed(self, path: str, old_entry: FileEntry,
+                      new_entry: FileEntry) -> bool:
+        """快速判断文件内容是否真正变更（mtime 可能被 git checkout 刷新）。"""
+        # 快速路径：size 不同则必然变更
+        if old_entry.size != new_entry.size:
+            return True
+        # 对 100KB 以内的文本文件做 hash 对比
+        if new_entry.size > 100 * 1024:
+            return True  # 大文件保守视为变更
+        ext = Path(path).suffix.lower()
+        if ext in BINARY_EXTS:
+            return True  # 二进制文件保守视为变更
+        try:
+            fp = self.project_root / path
+            content = fp.read_bytes()
+            new_hash = hashlib.sha256(content).hexdigest()
+            if old_entry.hash:
+                return old_entry.hash != new_hash
+            # 旧快照无 hash（首次迁移），计算旧 hash
+            hashlib.sha256(content).hexdigest()
+            new_entry.hash = new_hash
+            return True  # 无旧 hash 时保守视为变更
+        except Exception:
+            return True
+
+    # ---- 内容缓存 ----
+
+    def read_cached(self, path: str, cache: dict[str, str]) -> str | None:
+        """读取文件内容，优先使用缓存。"""
+        if path in cache:
+            return cache[path]
+        fp = self.project_root / path
+        if not fp.exists():
+            return None
+        try:
+            content = fp.read_text(encoding="utf-8", errors="replace")
+            cache[path] = content
+            return content
+        except Exception:
+            return None
+
+    def update_snapshot(self) -> None:
+        """更新快照为当前文件系统状态。在任务完成后调用。"""
+        entries = self.build_snapshot()
+        # 保留已有 hash 值避免重复计算
+        old = self.load()
+        if old:
+            for path, entry in entries.items():
+                if path in old and old[path].mtime == entry.mtime and old[path].size == entry.size:
+                    entry.hash = old[path].hash
+        self.save(entries)

ai_dev_flow/state.py

@@ -0,0 +1,87 @@
+"""状态文件管理 - .pipeline_stage / .pipeline_note 读写。"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+logger = logging.getLogger("ai-dev-flow")
+
+STAGE_FILE = ".pipeline_stage"
+NOTE_FILE = ".pipeline_note"
+TASK_STATE_FILE = "task_state.json"
+
+
+def read_stage(ai_dev_dir: Path) -> str | None:
+    """读取当前进度，无文件时返回 None。"""
+    sf = ai_dev_dir / STAGE_FILE
+    if not sf.exists():
+        return None
+    stage = sf.read_text(encoding="utf-8").strip()
+    logger.debug(f"read stage: {stage}")
+    return stage
+
+
+def write_stage(ai_dev_dir: Path, stage: str) -> None:
+    """写入当前进度。"""
+    sf = ai_dev_dir / STAGE_FILE
+    sf.write_text(stage, encoding="utf-8")
+    logger.debug(f"write stage: {stage}")
+
+
+def clear_stage(ai_dev_dir: Path) -> None:
+    """删除状态文件（管线全部完成后）。"""
+    sf = ai_dev_dir / STAGE_FILE
+    if sf.exists():
+        sf.unlink()
+        logger.debug("stage file removed")
+
+
+def read_note(ai_dev_dir: Path) -> str | None:
+    """读取上次运行留下的笔记。"""
+    nf = ai_dev_dir / NOTE_FILE
+    if not nf.exists():
+        return None
+    return nf.read_text(encoding="utf-8").strip()
+
+
+def write_note(ai_dev_dir: Path, note: str) -> None:
+    """写入笔记。"""
+    nf = ai_dev_dir / NOTE_FILE
+    nf.write_text(note, encoding="utf-8")
+    logger.info("note saved")
+
+
+def clear_note(ai_dev_dir: Path) -> None:
+    """删除笔记。"""
+    nf = ai_dev_dir / NOTE_FILE
+    if nf.exists():
+        nf.unlink()
+
+
+def read_task_state(ai_dev_dir: Path) -> dict | None:
+    """读取任务执行状态。返回 None 表示无状态文件。"""
+    import json
+    sf = ai_dev_dir / TASK_STATE_FILE
+    if not sf.exists():
+        return None
+    return json.loads(sf.read_text(encoding="utf-8"))
+
+
+def write_task_state(ai_dev_dir: Path, state: dict) -> None:
+    """原子写入任务执行状态（写临时文件后重命名）。"""
+    import json
+    import tempfile
+    sf = ai_dev_dir / TASK_STATE_FILE
+    tmp = tempfile.NamedTemporaryFile(
+        mode="w", encoding="utf-8", delete=False,
+        dir=ai_dev_dir, prefix=".task_state_tmp"
+    )
+    try:
+        json.dump(state, tmp, indent=2, ensure_ascii=False)
+        tmp.flush()
+        tmp.close()
+        Path(tmp.name).replace(sf)
+    except Exception:
+        Path(tmp.name).unlink(missing_ok=True)
+        raise