forgex-cli 0.1.0__py3-none-any.whl

forgex/context/chunker.py

@@ -0,0 +1,102 @@
+"""Chunking + selection (spec section 16: context compression)."""
+
+from __future__ import annotations
+
+import ast
+import re
+from dataclasses import dataclass
+
+_CAMEL_RE = re.compile(r"[A-Z]?[a-z]+|[A-Z]+(?![a-z])")
+
+
+@dataclass
+class Chunk:
+    name: str
+    text: str
+
+
+def chunk_python(source: str) -> list[Chunk]:
+    chunks: list[Chunk] = []
+    try:
+        tree = ast.parse(source)
+    except SyntaxError:
+        return [Chunk("whole-file", source)]
+
+    def node_end(node: ast.AST) -> int:
+        val = getattr(node, "end_lineno", None) or getattr(node, "lineno", 0)
+        return int(val) if val is not None else 0
+
+    def_nodes = [
+        n for n in tree.body
+        if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))
+    ]
+    first_def_line = min((n.lineno for n in def_nodes), default=len(source.splitlines()) + 1)
+    head_lines = source.splitlines()[: max(first_def_line - 1, 0)]
+    if any(line.strip() for line in head_lines):
+        chunks.append(Chunk("module-head", "\n".join(head_lines)))
+
+    for node in tree.body:
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+            lines = source.splitlines()
+            body = "\n".join(lines[node.lineno - 1 : node_end(node)])
+            kind = "class" if isinstance(node, ast.ClassDef) else "def"
+            chunks.append(
+                Chunk(node.name, f"# --- {kind} {node.name} (line {node.lineno}) ---\n{body}")
+            )
+
+    return chunks or [Chunk("whole-file", source)]
+
+
+def chunk_generic(source: str, window: int = 80) -> list[Chunk]:
+    lines = source.splitlines()
+    if len(lines) <= window:
+        return [Chunk("whole-file", source)]
+    return [
+        Chunk(
+            f"lines-{i + 1}-{min(i + window, len(lines))}",
+            f"# --- lines {i + 1}-{min(i + window, len(lines))} ---\n"
+            + "\n".join(lines[i : i + window]),
+        )
+        for i in range(0, len(lines), window)
+    ]
+
+
+def _tokens(text: str) -> set[str]:
+    out: set[str] = set()
+    for part in re.split(r"[^A-Za-z0-9]+", text.lower()):
+        out.update(p for p in part.split("_") if len(p) >= 3)
+    for token in re.findall(r"[A-Za-z]{3,}", text):
+        out.update(p.lower() for p in _CAMEL_RE.findall(token))
+    return out
+
+
+def _matches(chunk: Chunk, terms: set[str]) -> bool:
+    chunk_terms = _tokens(chunk.name) | _tokens(chunk.text[:2000])
+    return bool(chunk_terms & terms)
+
+
+def select_chunks(chunks: list[Chunk], terms: set[str], max_chars: int) -> tuple[str, bool]:
+    total = sum(len(chunk.text) for chunk in chunks)
+    if total <= max_chars:
+        return "\n\n".join(chunk.text for chunk in chunks), False
+
+    head = next((c for c in chunks if c.name == "module-head"), None)
+    ordered: list[Chunk] = []
+    if head is not None:
+        ordered.append(head)
+    rest = [c for c in chunks if c is not head]
+    ordered.extend(c for c in rest if _matches(c, terms))
+    ordered.extend(c for c in rest if not _matches(c, terms))
+
+    parts: list[str] = []
+    used = 0
+    for chunk in ordered:
+        if used >= max_chars:
+            break
+        text = chunk.text
+        if len(text) > max_chars - used:
+            text = text[: max_chars - used] + "\n... [chunk trimmed]"
+        parts.append(text)
+        used += len(text)
+
+    return "\n... [lines omitted] ...\n".join(parts), True

forgex/context/dependency_graph.py

@@ -0,0 +1,110 @@
+"""Import dependency graph (spec section 17)."""
+
+from __future__ import annotations
+
+import posixpath
+from collections import defaultdict, deque
+from typing import Any
+
+from forgex.context.indexer import IndexData
+
+
+def _match(files: dict[str, Any], candidates: list[str]) -> str | None:
+    for c in candidates:
+        if c in files:
+            return c
+    # Try searching suffix match across all project files
+    for c in candidates:
+        matching = [f for f in files if f.endswith("/" + c) or f == c]
+        if matching:
+            return min(matching, key=len)
+    return None
+
+
+def _resolve_python(imp: str, importer_rel: str, files: dict[str, Any]) -> str | None:
+    importer_dir = posixpath.dirname(importer_rel)
+    if imp.startswith("."):
+        # Relative import
+        level = len(imp) - len(imp.lstrip("."))
+        sub = imp.lstrip(".").replace(".", "/")
+        target_dir = importer_dir
+        for _ in range(level - 1):
+            target_dir = posixpath.dirname(target_dir)
+        base = posixpath.normpath(posixpath.join(target_dir, sub)) if sub else target_dir
+        return _match(files, [f"{base}.py", f"{base}/__init__.py"])
+
+    # Absolute import
+    base = imp.replace(".", "/")
+    return _match(files, [f"{base}.py", f"{base}/__init__.py"])
+
+
+def _resolve_js(imp: str, importer_rel: str, files: dict[str, Any]) -> str | None:
+    if not imp.startswith("."):
+        return None  # External package
+    importer_dir = posixpath.dirname(importer_rel)
+    base = posixpath.normpath(posixpath.join(importer_dir, imp))
+    candidates = [
+        base,
+        f"{base}.js",
+        f"{base}.jsx",
+        f"{base}.ts",
+        f"{base}.tsx",
+        f"{base}/index.js",
+        f"{base}/index.ts",
+    ]
+    return _match(files, candidates)
+
+
+class DependencyGraph:
+    def __init__(self, index: IndexData) -> None:
+        self._index = index
+        self._imports: dict[str, list[str]] = defaultdict(list)
+        self._importers: dict[str, list[str]] = defaultdict(list)
+        self._build_graph()
+
+    def _build_graph(self) -> None:
+        files = self._index.files
+        for rel_path, entry in files.items():
+            for raw_import in entry.imports:
+                target = None
+                if entry.language == "python":
+                    target = _resolve_python(raw_import, rel_path, files)
+                elif entry.language in ("javascript", "typescript"):
+                    target = _resolve_js(raw_import, rel_path, files)
+                if target and target != rel_path:
+                    if target not in self._imports[rel_path]:
+                        self._imports[rel_path].append(target)
+                    if rel_path not in self._importers[target]:
+                        self._importers[target].append(rel_path)
+
+    def imports_of(self, file: str, max_depth: int = 1) -> dict[str, int]:
+        results: dict[str, int] = {}
+        queue: deque[tuple[str, int]] = deque([(file, 0)])
+        seen = {file}
+
+        while queue:
+            curr, depth = queue.popleft()
+            if depth >= max_depth:
+                continue
+            for target in self._imports.get(curr, []):
+                if target not in seen:
+                    seen.add(target)
+                    results[target] = depth + 1
+                    queue.append((target, depth + 1))
+        return results
+
+    def importers_of(self, file: str, max_depth: int = 1) -> dict[str, int]:
+        results: dict[str, int] = {}
+        queue: deque[tuple[str, int]] = deque([(file, 0)])
+        seen = {file}
+
+        while queue:
+            curr, depth = queue.popleft()
+            if depth >= max_depth:
+                continue
+            for importer in self._importers.get(curr, []):
+                if importer not in seen:
+                    seen.add(importer)
+                    results[importer] = depth + 1
+                    queue.append((importer, depth + 1))
+        return results

forgex/context/indexer.py

@@ -0,0 +1,161 @@
+"""Project indexer: symbols, imports, and file metadata (spec section 17)."""
+
+from __future__ import annotations
+
+import ast
+import os
+import re
+from collections.abc import Iterator
+from datetime import UTC, datetime
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+from forgex.tools.filesystem import IGNORED_DIRS, is_secret_file
+
+INDEX_VERSION = 1
+MAX_PARSE_BYTES = 500_000
+MAX_SYMBOLS_PER_FILE = 100
+MAX_IMPORTS_PER_FILE = 50
+
+LANGUAGE_BY_SUFFIX = {
+    ".py": "python",
+    ".js": "javascript",
+    ".jsx": "javascript",
+    ".ts": "typescript",
+    ".tsx": "typescript",
+    ".rs": "rust",
+    ".go": "go",
+    ".java": "java",
+    ".json": "json",
+    ".toml": "toml",
+    ".yaml": "yaml",
+    ".yml": "yaml",
+    ".md": "markdown",
+    ".txt": "text",
+}
+
+PARSEABLE = {"python", "javascript", "typescript"}
+
+
+class SymbolInfo(BaseModel):
+    name: str
+    kind: str  # function | class | method | export
+    line: int
+
+
+class FileIndex(BaseModel):
+    size: int
+    mtime: float
+    language: str
+    symbols: list[SymbolInfo] = Field(default_factory=list)
+    imports: list[str] = Field(default_factory=list)
+
+
+class IndexData(BaseModel):
+    version: int = INDEX_VERSION
+    generated_at: str
+    files: dict[str, FileIndex] = Field(default_factory=dict)
+    stats: dict[str, int] = Field(default_factory=dict)
+
+
+def iter_project_files(workspace: Path) -> Iterator[Path]:
+    for root, dirs, files in os.walk(workspace):
+        # Mutate dirs in place to prevent descending into ignored dirs
+        dirs[:] = [d for d in dirs if d not in IGNORED_DIRS]
+        for f in files:
+            if is_secret_file(f):
+                continue
+            path = Path(root) / f
+            yield path
+
+
+def _python_parse(source: str) -> tuple[list[SymbolInfo], list[str]]:
+    symbols: list[SymbolInfo] = []
+    imports: list[str] = []
+    try:
+        tree = ast.parse(source)
+    except SyntaxError:
+        return [], []
+
+    for node in tree.body:
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            symbols.append(SymbolInfo(name=node.name, kind="function", line=node.lineno))
+        elif isinstance(node, ast.ClassDef):
+            symbols.append(SymbolInfo(name=node.name, kind="class", line=node.lineno))
+            for child in node.body:
+                if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                    symbols.append(
+                        SymbolInfo(name=f"{node.name}.{child.name}", kind="method", line=child.lineno)
+                    )
+
+    for item in ast.walk(tree):
+        if isinstance(item, ast.Import):
+            imports.extend(alias.name for alias in item.names)
+        elif isinstance(item, ast.ImportFrom):
+            prefix = "." * item.level
+            if item.module:
+                imports.append(prefix + item.module)
+            else:
+                for alias in item.names:
+                    imports.append(prefix + alias.name)
+
+    return symbols[:MAX_SYMBOLS_PER_FILE], imports[:MAX_IMPORTS_PER_FILE]
+
+
+_JS_IMPORT_RE = re.compile(
+    r"""(?:(?:import|export)\s+[^;\n]*?\s+from\s+|import\s+|require\s*\(\s*|import\s*\(\s*)['"]([^'"]+)['"]"""
+)
+_JS_SYMBOL_RE = re.compile(
+    r"export\s+(?:default\s+)?(?:async\s+)?(?:function|class|const|let)\s+(\w+)"
+)
+
+
+def _js_parse(source: str) -> tuple[list[SymbolInfo], list[str]]:
+    imports = _JS_IMPORT_RE.findall(source)[:MAX_IMPORTS_PER_FILE]
+    symbols = [
+        SymbolInfo(name=name, kind="export", line=0)
+        for name in _JS_SYMBOL_RE.findall(source)[:MAX_SYMBOLS_PER_FILE]
+    ]
+    return symbols, imports
+
+
+def index_file(path: Path, rel: str, size: int, mtime: float) -> FileIndex:
+    language = LANGUAGE_BY_SUFFIX.get(path.suffix.lower(), "other")
+    entry = FileIndex(size=size, mtime=mtime, language=language)
+    if language in PARSEABLE and size <= MAX_PARSE_BYTES:
+        try:
+            source = path.read_text(encoding="utf-8", errors="replace")
+        except OSError:
+            return entry
+        symbols, imports = _python_parse(source) if language == "python" else _js_parse(source)
+        entry.symbols, entry.imports = symbols, imports
+    return entry
+
+
+def build(workspace: Path, cached: IndexData | None = None) -> IndexData:
+    workspace = workspace.resolve()
+    cached_files = cached.files if cached and cached.version == INDEX_VERSION else {}
+    files: dict[str, FileIndex] = {}
+    parsed = reused = 0
+
+    for path in iter_project_files(workspace):
+        rel = path.relative_to(workspace).as_posix()
+        try:
+            stat = path.stat()
+        except OSError:
+            continue
+        old = cached_files.get(rel)
+        if old is not None and old.size == stat.st_size and old.mtime == stat.st_mtime:
+            files[rel] = old
+            reused += 1
+            continue
+        files[rel] = index_file(path, rel, stat.st_size, stat.st_mtime)
+        parsed += 1
+
+    pruned = max(0, len(cached_files) - reused)
+    return IndexData(
+        generated_at=datetime.now(UTC).isoformat(),
+        files=files,
+        stats={"parsed": parsed, "reused": reused, "pruned": pruned},
+    )

forgex/context/manager.py

@@ -0,0 +1,174 @@
+"""ContextManager (spec section 17): index → retrieve → compress → assemble.
+
+Fail-open by design in callers: context intelligence enhances prompts but
+must never block a build. The index cache lives in .forgex/cache/.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from forgex.context import indexer
+from forgex.context.chunker import chunk_generic, chunk_python, select_chunks
+from forgex.context.dependency_graph import DependencyGraph
+from forgex.context.indexer import IndexData
+from forgex.context.retriever import ScoredFile, query_terms, score_files
+from forgex.utils.paths import PathViolationError, resolve_in_workspace
+from forgex.utils.redact import redact_patterns
+
+logger = logging.getLogger("forgex.context")
+
+
+class ContextFile(BaseModel):
+    path: str
+    content: str
+    truncated: bool = False
+    reasons: list[str] = Field(default_factory=list)
+
+
+class ContextBundle(BaseModel):
+    files: list[ContextFile] = Field(default_factory=list)
+    stats: dict[str, Any] = Field(default_factory=dict)
+
+    def render(self) -> str:
+        sections = []
+        for f in self.files:
+            trunc = " (compressed)" if f.truncated else ""
+            sections.append(f"--- {f.path}{trunc} ---\n{f.content}")
+        return "\n\n".join(sections)
+
+
+class ContextManager:
+    CACHE_FILE = "context-index.json"
+
+    def __init__(self, workspace: Path) -> None:
+        self.workspace = Path(workspace).resolve()
+        self._cache_path = self.workspace / ".forgex" / "cache" / self.CACHE_FILE
+        self._index: IndexData | None = None
+        self._graph: DependencyGraph | None = None
+
+    def _load_cache(self) -> IndexData | None:
+        if not self._cache_path.is_file():
+            return None
+        try:
+            return IndexData.model_validate_json(self._cache_path.read_text(encoding="utf-8"))
+        except Exception:  # noqa: BLE001
+            return None
+
+    def _save_cache(self, index: IndexData) -> None:
+        try:
+            self._cache_path.parent.mkdir(parents=True, exist_ok=True)
+            self._cache_path.write_text(index.model_dump_json(), encoding="utf-8")
+        except OSError:
+            pass
+
+    def ensure_index(self, refresh: bool = False) -> IndexData:
+        if self._index is not None and not refresh:
+            return self._index
+
+        cached = None if refresh else self._load_cache()
+        index = indexer.build(self.workspace, cached)
+        self._save_cache(index)
+        self._index = index
+        self._graph = DependencyGraph(index)
+        return index
+
+    @property
+    def graph(self) -> DependencyGraph:
+        if self._graph is None:
+            self.ensure_index()
+        assert self._graph is not None
+        return self._graph
+
+    def symbol_map(self) -> str:
+        index = self.ensure_index()
+        lines: list[str] = []
+        for path, entry in sorted(index.files.items()):
+            if entry.symbols:
+                names = [f"{s.kind} {s.name}" if s.kind in ("class", "def") else s.name for s in entry.symbols[:10]]
+                lines.append(f"{path}: {', '.join(names)}")
+        return "\n".join(lines)
+
+    def relevant_files(
+        self,
+        query: str,
+        seeds: list[str],
+        recent: list[str] | None = None,
+        limit: int = 10,
+    ) -> list[ScoredFile]:
+        index = self.ensure_index()
+        return score_files(
+            index=index,
+            graph=self.graph,
+            query=query,
+            seeds=seeds,
+            recent=recent,
+            limit=limit,
+        )
+
+    def importers_of(self, file: str, max_depth: int = 1) -> list[str]:
+        self.ensure_index()
+        return sorted(self.graph.importers_of(file, max_depth=max_depth).keys())
+
+    def build_context(
+        self,
+        query: str,
+        seeds: list[str],
+        budget: int,
+        max_files: int = 4,
+        exclude: set[str] | None = None,
+    ) -> ContextBundle:
+        ranked = self.relevant_files(query=query, seeds=seeds, limit=max_files * 2)
+        exclude_set = exclude or set()
+
+        files: list[ContextFile] = []
+        remaining_budget = budget
+        terms = set(query_terms(query))
+
+        for item in ranked:
+            if item.path in exclude_set:
+                continue
+            if len(files) >= max_files or remaining_budget <= 0:
+                break
+
+            try:
+                real_path = resolve_in_workspace(self.workspace, item.path)
+            except PathViolationError:
+                continue
+
+            if not real_path.is_file():
+                continue
+
+            try:
+                source = real_path.read_text(encoding="utf-8", errors="replace")
+            except OSError:
+                continue
+
+            source = redact_patterns(source)
+
+            # Chunk and select
+            per_file_budget = min(remaining_budget, budget // 2 if len(ranked) > 1 else budget)
+            if real_path.suffix.lower() == ".py":
+                chunks = chunk_python(source)
+            else:
+                chunks = chunk_generic(source)
+
+            content, truncated = select_chunks(chunks, terms, max_chars=per_file_budget)
+            files.append(
+                ContextFile(
+                    path=item.path,
+                    content=content,
+                    truncated=truncated,
+                    reasons=item.reasons,
+                )
+            )
+            remaining_budget -= len(content)
+
+        return ContextBundle(
+            files=files,
+            stats={"budget_used": budget - remaining_budget, "index_files": len(self.ensure_index().files)},
+        )

forgex/context/retriever.py

@@ -0,0 +1,99 @@
+"""Lexical relevance retrieval with auditable reasons (spec section 16)."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+from forgex.context.dependency_graph import DependencyGraph
+from forgex.context.indexer import IndexData
+
+STOPWORDS = {
+    "the", "and", "for", "with", "into", "from", "this", "that", "must",
+    "should", "will", "your", "are", "all", "not", "but", "can", "has",
+    "have", "when", "which", "was", "you", "its", "our",
+}
+_CAMEL_RE = re.compile(r"[A-Z]?[a-z]+|[A-Z]+(?![a-z])")
+
+
+def query_terms(text: str, *, limit: int = 12) -> list[str]:
+    terms: list[str] = []
+    for token in re.findall(r"[A-Za-z][A-Za-z0-9_]{2,}", text):
+        for part in _CAMEL_RE.findall(token.replace("_", " ")) or [token]:
+            p = part.lower()
+            if len(p) >= 3 and p not in STOPWORDS and p not in terms:
+                terms.append(p)
+        snake = token.lower().replace("_", " ").split()
+        for p in snake:
+            if len(p) >= 3 and p not in STOPWORDS and p not in terms:
+                terms.append(p)
+    return terms[:limit]
+
+
+@dataclass
+class ScoredFile:
+    path: str
+    score: int = 0
+    reasons: list[str] = field(default_factory=list)
+
+    def bump(self, points: int, reason: str) -> None:
+        self.score += points
+        self.reasons.append(reason)
+
+
+def _path_tokens(path: str) -> set[str]:
+    return {t for t in re.split(r"[^a-z0-9]+", path.lower()) if len(t) >= 3}
+
+
+def _is_test_path(path: str) -> bool:
+    return "test" in path.lower()
+
+
+def score_files(
+    *,
+    index: IndexData,
+    graph: DependencyGraph,
+    query: str,
+    seeds: list[str],
+    recent: list[str] | None = None,
+    limit: int = 20,
+) -> list[ScoredFile]:
+    scores: dict[str, ScoredFile] = {}
+
+    def get(path: str) -> ScoredFile:
+        return scores.setdefault(path, ScoredFile(path=path))
+
+    terms = set(query_terms(query))
+
+    for seed in seeds:
+        if seed in index.files:
+            get(seed).bump(100, "seed file (planned/changed)")
+        for importer, depth in graph.importers_of(seed, max_depth=2).items():
+            if importer in index.files:
+                points = 50 if depth == 1 else 15
+                get(importer).bump(points, f"imports seed ({'direct' if depth == 1 else 'transitive'})")
+        for dependency in graph.imports_of(seed, max_depth=1):
+            if dependency in index.files:
+                get(dependency).bump(30, "imported by seed")
+        seed_tokens = _path_tokens(seed)
+        if seed_tokens:
+            for path in index.files:
+                if _is_test_path(path) and _path_tokens(path) & seed_tokens:
+                    get(path).bump(30, f"test file for {seed}")
+
+    for path in recent or []:
+        if path in index.files:
+            get(path).bump(20, "recently changed")
+
+    for path, entry in index.files.items():
+        lowered = path.lower()
+        for term in terms:
+            if term in lowered:
+                get(path).bump(5, f"path matches '{term}'")
+            for symbol in entry.symbols:
+                if term in symbol.name.lower():
+                    get(path).bump(10, f"symbol '{symbol.name}' matches '{term}'")
+                    break
+
+    ranked = sorted(scores.values(), key=lambda s: (-s.score, s.path))
+    return [s for s in ranked if s.score > 0][:limit]

forgex/models/__init__.py

@@ -0,0 +1,25 @@
+"""Model layer: provider contract, capabilities, registry."""
+
+from forgex.models.base import (
+    Message,
+    ModelProvider,
+    ModelRequest,
+    ModelResponse,
+    ModelUsage,
+    ProviderHealth,
+    ToolCall,
+)
+from forgex.models.capabilities import ModelCapabilities
+from forgex.models.registry import ModelRegistry
+
+__all__ = [
+    "Message",
+    "ModelCapabilities",
+    "ModelProvider",
+    "ModelRegistry",
+    "ModelRequest",
+    "ModelResponse",
+    "ModelUsage",
+    "ProviderHealth",
+    "ToolCall",
+]