source-kb 0.2.2__py3-none-any.whl

core/prompt/content.py

@@ -0,0 +1,320 @@
+"""Prompt content helpers — shared utilities for variable computation.
+
+Computes template variables: high_methods, generated_docs, sibling_modules,
+prior_docs_context, shard_context, and source content reading.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def compute_high_methods(module_dir: Path) -> str:
+    """Extract high-complexity methods from skeleton summary for prompt injection."""
+    from core.paths import resolve_skeleton_summary
+    summary_file = resolve_skeleton_summary(module_dir)
+    if summary_file is None:
+        return "(Skeleton unavailable, identify complex methods from source code)"
+    try:
+        entries = json.loads(summary_file.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return "(Failed to read skeleton)"
+
+    high_methods: list[str] = []
+    for entry in entries:
+        file_path = entry.get("file", "")
+        classname = Path(file_path).stem if file_path else "Unknown"
+        for m in entry.get("high_complexity_methods", []):
+            name = m.get("name", "") if isinstance(m, dict) else str(m)
+            line_count = m.get("line_count", "?") if isinstance(m, dict) else "?"
+            high_methods.append(f"- {classname}.{name} ({line_count} lines)")
+        if not entry.get("high_complexity_methods"):
+            for m in entry.get("methods", []):
+                if m.get("complexity") == "high":
+                    high_methods.append(f"- {classname}.{m['name']} ({m.get('line_count', '?')} lines)")
+    if not high_methods:
+        return "(No high-complexity methods)"
+    return "\n".join(high_methods[:30])
+
+
+def scan_generated_docs(module_dir: Path) -> str:
+    """List already-generated .md files in the module directory."""
+    if not module_dir.is_dir():
+        return "(No generated documents yet)"
+    docs = sorted(f.name for f in module_dir.glob("*.md") if not f.name.startswith("."))
+    if not docs:
+        return "(No generated documents yet)"
+    return "\n".join(f"- [{doc}](./{doc})" for doc in docs)
+
+
+def compute_sibling_modules(config: dict[str, Any], kb_name: str, current_module: str) -> str:
+    """Compute sibling module info for cross-module awareness."""
+    kb = config.get("knowledge_bases", {}).get(kb_name, {})
+    source = kb.get("source", {})
+    siblings: list[str] = []
+    if source.get("structure") == "multi-repo":
+        for repo in source.get("repos", []):
+            name = repo.get("name", "")
+            if name and name != current_module:
+                siblings.append(f"- **{name}** ({repo.get('description', repo.get('type', ''))})")
+    elif source.get("structure") == "monorepo":
+        for mod in source.get("modules", []):
+            name = mod.get("name", "")
+            if name and name != current_module:
+                siblings.append(f"- **{name}** ({mod.get('type', '')})")
+    if not siblings:
+        return "(No sibling modules)"
+    return "### Sibling modules in the same knowledge base\n\n" + "\n".join(siblings)
+
+
+def build_prior_docs_context(
+    module_dir: Path, current_doc_type: str, max_chars: int = 2000,
+    preset: dict | None = None,
+) -> str:
+    """Build context from prior-batch documents for cross-doc-type reference.
+
+    Dependencies are loaded from preset doc_types.yaml `depends_on` field.
+    Falls back to hardcoded defaults if preset is not provided.
+
+    Note: max_chars default (2000) can be overridden via preset limits.prior_docs_max_chars
+    """
+    # Use configured limit if available
+    if preset and max_chars == 2000:
+        limits = preset.get("limits", {})
+        max_chars = limits.get("prior_docs_max_chars", max_chars)
+    dep_docs = _get_dependencies(current_doc_type, preset)
+    if not dep_docs:
+        return ""
+    summaries: list[str] = []
+    remaining = max_chars
+    for dep_type in dep_docs:
+        filename = _doc_type_to_filename(dep_type)
+        doc_path = module_dir / filename
+        if not doc_path.exists():
+            continue
+        summary = _extract_doc_summary(doc_path)
+        if not summary:
+            continue
+        section = f"### {filename}\n{summary}"
+        if len(section) > remaining:
+            break
+        summaries.append(section)
+        remaining -= len(section)
+    if not summaries:
+        return ""
+    return "## Prior document summaries (available for reference)\n\n" + "\n\n".join(summaries)
+
+
+def _get_dependencies(doc_type: str, preset: dict | None) -> list[str]:
+    """Get dependency doc types from preset config."""
+    if preset:
+        from core.preset import get_doc_type_config
+        cfg = get_doc_type_config(preset, doc_type)
+        deps = cfg.get("depends_on", [])
+        if deps:
+            return deps
+    logger.debug("no depends_on for doc_type=%s, skipping prior docs", doc_type)
+    return []
+
+
+def read_source_content(
+    module_dir: Path, doc_type: str, source_cache: Path, max_bytes: int = 300_000,
+) -> str:
+    """Read source files from file list, priority-sorted, with byte-limit truncation.
+
+    Priority: high-complexity files first, then by line count, then alphabetical.
+    """
+    from core.paths import resolve_file_list
+    fl_path = resolve_file_list(module_dir, doc_type)
+    if fl_path is None or not fl_path.exists():
+        return ""
+    file_paths = [l.strip() for l in fl_path.read_text(encoding="utf-8").splitlines()
+                  if l.strip() and not l.strip().startswith("#")]
+    if not file_paths:
+        return ""
+
+    priority_map = _build_priority_map(module_dir)
+    source_cache_str = str(source_cache.resolve())
+
+    def sort_key(fpath: str):
+        # fpath is now module-relative (e.g. "promotion-manager-api/src/main/java/...")
+        # For backward compat, also strip source_cache prefix if present (old absolute paths)
+        rel = fpath
+        if fpath.startswith(source_cache_str):
+            rel = fpath[len(source_cache_str):].lstrip("/\\")
+        info = priority_map.get(rel, {})
+        return (-info.get("high_methods", 0), -info.get("total_lines", 0), fpath)
+
+    file_paths.sort(key=sort_key)
+    parts: list[str] = []
+    total_bytes_used = 0
+    included = 0
+    max_single_file = max_bytes // 6
+
+    for fpath in file_paths:
+        src_file = source_cache / fpath if not Path(fpath).is_absolute() else Path(fpath)
+        if not src_file.exists():
+            continue
+        try:
+            raw_content = src_file.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError):
+            continue
+        content_bytes = len(raw_content.encode("utf-8"))
+        if content_bytes <= max_single_file:
+            content = raw_content
+        else:
+            lines = raw_content.splitlines(keepends=True)
+            truncated: list[str] = []
+            acc = 0
+            for line in lines:
+                acc += len(line.encode("utf-8"))
+                if acc > max_single_file:
+                    break
+                truncated.append(line)
+            content = "".join(truncated)
+            content += f"\n// ... [truncated, showing {len(truncated)}/{len(lines)} lines]\n"
+
+        ext = Path(fpath).suffix.lstrip(".") or "text"
+        block = f"### {Path(fpath).name}\n```{ext}\n{content}\n```\n"
+        block_bytes = len(block.encode("utf-8"))
+        if total_bytes_used + block_bytes > max_bytes:
+            parts.append(f"\n[truncated — {len(file_paths) - included} files omitted]\n")
+            break
+        parts.append(block)
+        total_bytes_used += block_bytes
+        included += 1
+
+    return "\n".join(parts)
+
+
+def _build_priority_map(module_dir: Path) -> dict[str, dict]:
+    """Build file priority map from skeleton: {relative_path: {high_methods, total_lines}}."""
+    from core.paths import resolve_skeleton, resolve_skeleton_summary
+    entries: list[dict] = []
+    summary_file = resolve_skeleton_summary(module_dir)
+    if summary_file is not None:
+        try:
+            entries = json.loads(summary_file.read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            pass
+    if not entries:
+        resolved = resolve_skeleton(module_dir)
+        if resolved is not None:
+            if resolved.is_dir():
+                for f in resolved.glob("*.json"):
+                    try:
+                        data = json.loads(f.read_text(encoding="utf-8"))
+                        if isinstance(data, list):
+                            entries.extend(data)
+                    except (OSError, ValueError):
+                        continue
+            else:
+                try:
+                    data = json.loads(resolved.read_text(encoding="utf-8"))
+                    entries = data if isinstance(data, list) else []
+                except (OSError, ValueError):
+                    pass
+    priority: dict[str, dict] = {}
+    for entry in entries:
+        fpath = entry.get("file", "")
+        if not fpath:
+            continue
+        high_count = len(entry.get("high_complexity_methods", []))
+        if not high_count:
+            high_count = sum(1 for m in entry.get("methods", [])
+                            if isinstance(m, dict) and m.get("complexity") == "high")
+        priority[fpath] = {"high_methods": high_count, "total_lines": entry.get("line_count", 0)}
+    return priority
+
+
+def build_shard_context(
+    module_dir: Path, doc_type: str, current_shard: str,
+    completed_shards: list[str], max_chars: int = 1500,
+    preset: dict | None = None,
+) -> str:
+    """Build context from completed shards of the same doc type.
+
+    Note: max_chars default (1500) can be overridden via preset limits.shard_context_max_chars
+    """
+    if not completed_shards:
+        return ""
+    # Use configured limit if available
+    if preset and max_chars == 1500:
+        limits = preset.get("limits", {})
+        max_chars = limits.get("shard_context_max_chars", max_chars)
+    doc_basename = _doc_type_to_filename(doc_type).replace(".md", "")
+    parts: list[str] = []
+    for shard_name in completed_shards:
+        if shard_name == current_shard:
+            continue
+        shard_path = module_dir / f"{doc_basename}-{shard_name}.md"
+        if not shard_path.exists():
+            continue
+        summary = _extract_shard_summary(shard_path)
+        if summary:
+            parts.append(f"### {shard_name}\n{summary}")
+    if not parts:
+        return ""
+    text = "## Content already covered by prior shards (avoid repetition)\n\n" + "\n\n".join(parts)
+    if len(text) > max_chars:
+        text = text[:max_chars - 20] + "\n\n[truncated]"
+    return text
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+_DOC_TYPE_FILENAMES: dict[str, str] = {}
+
+
+def _doc_type_to_filename(doc_type: str, preset: dict | None = None) -> str:
+    """Convert doc_type key to filename."""
+    if preset:
+        try:
+            from core.preset import get_doc_type_config
+            dt_config = get_doc_type_config(preset, doc_type)
+            if dt_config and "filename" in dt_config:
+                return dt_config["filename"]
+        except (ImportError, Exception):
+            pass
+    return f"{doc_type}.md"
+
+
+def _extract_doc_summary(path: Path, max_lines: int = 15) -> str:
+    """Extract title + heading list from a document."""
+    try:
+        content = path.read_text(encoding="utf-8")
+    except (OSError, UnicodeDecodeError):
+        return ""
+    lines = content.splitlines()
+    title = ""
+    for line in lines[:5]:
+        if line.startswith("# ") and not line.startswith("## "):
+            title = line.strip()
+            break
+    headings = [l.strip().lstrip("#").strip() for l in lines if l.strip().startswith("## ")]
+    parts = []
+    if title:
+        parts.append(title)
+    if headings:
+        parts.append("Sections: " + " | ".join(headings[:10]))
+    return "\n".join(parts)
+
+
+def _extract_shard_summary(path: Path) -> str:
+    """Extract heading list from a shard document."""
+    try:
+        content = path.read_text(encoding="utf-8")
+    except (OSError, UnicodeDecodeError):
+        return ""
+    headings = [l.strip() for l in content.splitlines() if l.strip().startswith("## ")]
+    if headings:
+        return "Sections: " + " | ".join(h.lstrip("#").strip() for h in headings[:8])
+    return ""

core/prompt/context_manager.py

@@ -0,0 +1,164 @@
+"""Prompt context size management.
+
+Ensures prompts don't exceed model context limits by applying
+progressive truncation strategies.
+
+Usage:
+    from core.prompt.context_manager import estimate_tokens, manage_context_size
+
+    system, user = manage_context_size(system, user, max_tokens=128000)
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+
+logger = logging.getLogger(__name__)
+
+
+def estimate_tokens(text: str) -> int:
+    """Fast token count approximation.
+
+    Rules:
+    - Chinese characters: ~1.5 tokens per char (conservative)
+    - English/code: ~4 chars per token
+    - Mixed: weighted by character type
+    """
+    if not text:
+        return 0
+
+    chinese_chars = len(re.findall(r'[\u4e00-\u9fff]', text))
+    other_chars = len(text) - chinese_chars
+
+    chinese_tokens = int(chinese_chars * 1.5)
+    other_tokens = other_chars // 4
+
+    return chinese_tokens + other_tokens
+
+
+def manage_context_size(
+    system: str,
+    user: str,
+    *,
+    max_tokens: int = 128_000,
+    threshold_ratio: float = 0.8,
+) -> tuple[str, str]:
+    """Apply progressive truncation if prompt exceeds threshold.
+
+    Truncation priority (applied in order until under limit):
+    1. Truncate skeleton/skeleton_delta sections in user prompt
+    2. Truncate source_code/source_snippets sections
+    3. Truncate old_content to heading list + first 3 lines per section
+
+    Args:
+        system: System prompt
+        user: User prompt
+        max_tokens: Maximum context tokens
+        threshold_ratio: Trigger truncation at this ratio of max
+
+    Returns:
+        (system, user) — possibly truncated
+    """
+    threshold = int(max_tokens * threshold_ratio)
+    total = estimate_tokens(system) + estimate_tokens(user)
+
+    if total <= threshold:
+        return system, user
+
+    original_total = total
+    truncated_what: list[str] = []
+
+    # Strategy 1: Truncate skeleton sections
+    user, reduced = _truncate_section(user, "skeleton", max_lines=100)
+    if reduced:
+        truncated_what.append("skeleton")
+        total = estimate_tokens(system) + estimate_tokens(user)
+        if total <= threshold:
+            _log_truncation(original_total, total, truncated_what)
+            return system, user
+
+    user, reduced = _truncate_section(user, "skeleton", max_lines=100)
+    if reduced:
+        truncated_what.append("skeleton_delta")
+        total = estimate_tokens(system) + estimate_tokens(user)
+        if total <= threshold:
+            _log_truncation(original_total, total, truncated_what)
+            return system, user
+
+    # Strategy 2: Truncate source code sections
+    user, reduced = _truncate_section(user, "source", max_lines=80)
+    if reduced:
+        truncated_what.append("source_code")
+        total = estimate_tokens(system) + estimate_tokens(user)
+        if total <= threshold:
+            _log_truncation(original_total, total, truncated_what)
+            return system, user
+
+    user, reduced = _truncate_section(user, "source", max_lines=80)
+    if reduced:
+        truncated_what.append("source_files")
+        total = estimate_tokens(system) + estimate_tokens(user)
+        if total <= threshold:
+            _log_truncation(original_total, total, truncated_what)
+            return system, user
+
+    # Strategy 3: Hard truncate user prompt to fit
+    max_user_tokens = threshold - estimate_tokens(system) - 500  # leave margin
+    if max_user_tokens > 0:
+        user = _hard_truncate(user, max_user_tokens)
+        truncated_what.append("hard_truncate")
+
+    _log_truncation(original_total, estimate_tokens(system) + estimate_tokens(user), truncated_what)
+    return system, user
+
+
+def _truncate_section(text: str, marker: str, max_lines: int = 100) -> tuple[str, bool]:
+    """Find a section containing marker keyword and truncate it."""
+    lines = text.split("\n")
+    marker_lower = marker.lower()
+
+    # Find section start (line containing marker)
+    start_idx = None
+    for i, line in enumerate(lines):
+        if marker_lower in line.lower() and (line.startswith("#") or line.endswith(":")):
+            start_idx = i
+            break
+
+    if start_idx is None:
+        return text, False
+
+    # Find section end (next heading or end)
+    end_idx = len(lines)
+    for j in range(start_idx + 1, len(lines)):
+        if lines[j].startswith("#") or (lines[j].strip() and lines[j][0].isalpha() and lines[j].endswith(":")):
+            end_idx = j
+            break
+
+    section_lines = lines[start_idx:end_idx]
+    if len(section_lines) <= max_lines:
+        return text, False
+
+    # Truncate section
+    truncated = section_lines[:max_lines]
+    truncated.append(f"\n... (truncated, {len(section_lines) - max_lines} lines removed)")
+
+    new_lines = lines[:start_idx] + truncated + lines[end_idx:]
+    return "\n".join(new_lines), True
+
+
+def _hard_truncate(text: str, max_tokens: int) -> str:
+    """Hard truncate text to approximately max_tokens."""
+    # Rough: 1 token ≈ 3 chars for mixed content
+    max_chars = max_tokens * 3
+    if len(text) <= max_chars:
+        return text
+    return text[:max_chars] + "\n\n... (truncated to fit context window)"
+
+
+def _log_truncation(original: int, final: int, what: list[str]) -> None:
+    """Log truncation details."""
+    logger.warning(
+        "[context_manager] Truncated prompt: %d → %d tokens (removed: %s)",
+        original, final, ", ".join(what),
+    )