dotscope 0.1.0__py3-none-any.whl

dotscope/passes/virtual.py

@@ -0,0 +1,239 @@
+"""Virtual scopes: cross-cutting concern detection from import graph hubs.
+
+Directory scopes capture physical structure. Virtual scopes capture logical
+architecture — a User lifecycle spanning models/, auth/, validators/, serializers/.
+
+Detection algorithm:
+1. Find hub files (imported by 3+ files from 2+ directories)
+2. Collect cluster (hub + importers + shared imports within 1 hop)
+3. Filter by cohesion (more internal edges than external)
+4. Name by centrality (most-imported symbol)
+5. Deduplicate overlapping clusters (>70% overlap → merge)
+"""
+
+import os
+from collections import defaultdict
+from pathlib import Path
+from typing import Dict, List, Optional, Set
+
+from ..context import parse_context
+from ..graph import DependencyGraph
+from ..models.core import ScopeConfig
+from ..models.passes import VirtualScope  # noqa: F401
+from ..tokens import estimate_scope_tokens
+
+# Utility directories whose files connect everything (not meaningful clusters)
+_UTILITY_DIRS = {"utils", "helpers", "common", "shared", "lib", "core"}
+
+
+def detect_virtual_scopes(
+    graph: DependencyGraph,
+    min_importers: int = 3,
+    min_directories: int = 2,
+    min_cohesion: float = 0.3,
+) -> List[ScopeConfig]:
+    """Detect cross-cutting concerns from the import graph.
+
+    Returns ScopeConfig objects for virtual scopes, ready to be
+    added to the ingest plan alongside directory scopes.
+    """
+    root = graph.root
+    hubs = _find_hubs(graph, min_importers, min_directories)
+    clusters = [_build_cluster(hub, graph) for hub in hubs]
+    clusters = [c for c in clusters if c.cohesion >= min_cohesion]
+    clusters = _deduplicate(clusters)
+
+    scopes = []
+    for cluster in clusters:
+        config = _cluster_to_scope(cluster, root)
+        if config:
+            scopes.append(config)
+
+    return scopes
+
+
+def _find_hubs(
+    graph: DependencyGraph, min_importers: int, min_dirs: int
+) -> List[str]:
+    """Find files imported by 3+ files from 2+ different directories."""
+    hubs = []
+    for path, node in graph.files.items():
+        if not node.imported_by:
+            continue
+
+        # Skip utility directories
+        parts = Path(path).parts
+        if len(parts) > 1 and parts[0].lower() in _UTILITY_DIRS:
+            continue
+
+        importer_dirs = set()
+        for imp_by in node.imported_by:
+            imp_parts = Path(imp_by).parts
+            if len(imp_parts) > 1:
+                importer_dirs.add(imp_parts[0])
+
+        if len(node.imported_by) >= min_importers and len(importer_dirs) >= min_dirs:
+            hubs.append(path)
+
+    return hubs
+
+
+def _build_cluster(hub: str, graph: DependencyGraph) -> VirtualScope:
+    """Build a cluster around a hub file.
+
+    Cluster = hub + all importers + shared imports within 1 hop.
+    """
+    hub_node = graph.files.get(hub)
+    if not hub_node:
+        return VirtualScope(name="", hub_file=hub, files=[], cohesion=0, directories_spanned=0)
+
+    cluster_files: Set[str] = {hub}
+    cluster_files.update(hub_node.imported_by)
+
+    # Add shared imports (files that multiple importers also import)
+    import_counts: Dict[str, int] = defaultdict(int)
+    for importer in hub_node.imported_by:
+        imp_node = graph.files.get(importer)
+        if imp_node:
+            for dep in imp_node.imports:
+                if dep != hub and dep not in cluster_files:
+                    import_counts[dep] += 1
+
+    # Only add shared imports that 2+ importers share
+    for dep, count in import_counts.items():
+        if count >= 2:
+            cluster_files.add(dep)
+
+    # Compute cohesion
+    internal_edges = 0
+    external_edges = 0
+    for f in cluster_files:
+        node = graph.files.get(f)
+        if not node:
+            continue
+        for imp in node.imports:
+            if imp in cluster_files:
+                internal_edges += 1
+            else:
+                external_edges += 1
+
+    total = internal_edges + external_edges
+    cohesion = internal_edges / total if total > 0 else 0.0
+
+    # Count directories spanned
+    dirs = set()
+    for f in cluster_files:
+        parts = Path(f).parts
+        if len(parts) > 1:
+            dirs.add(parts[0])
+
+    # Name from hub file
+    name = _infer_name(hub, graph)
+
+    return VirtualScope(
+        name=name,
+        hub_file=hub,
+        files=sorted(cluster_files),
+        cohesion=round(cohesion, 3),
+        directories_spanned=len(dirs),
+    )
+
+
+def _infer_name(hub: str, graph: DependencyGraph) -> str:
+    """Infer a name for the virtual scope from the hub file."""
+    basename = os.path.splitext(os.path.basename(hub))[0]
+    # e.g., "models/user.py" → "user_lifecycle"
+    if basename in ("__init__", "index"):
+        parts = Path(hub).parts
+        if len(parts) > 1:
+            basename = parts[-2]
+    return f"{basename}_lifecycle"
+
+
+def _deduplicate(clusters: List[VirtualScope]) -> List[VirtualScope]:
+    """Merge clusters with >70% file overlap."""
+    if len(clusters) <= 1:
+        return clusters
+
+    result = []
+    merged = set()
+
+    for i, a in enumerate(clusters):
+        if i in merged:
+            continue
+        best = a
+        for j, b in enumerate(clusters):
+            if j <= i or j in merged:
+                continue
+            a_set = set(a.files)
+            b_set = set(b.files)
+            overlap = len(a_set & b_set) / len(a_set | b_set) if (a_set | b_set) else 0
+            if overlap > 0.7:
+                # Keep the one with more files
+                if len(b.files) > len(best.files):
+                    best = b
+                merged.add(j)
+        result.append(best)
+
+    return result
+
+
+def _cluster_to_scope(cluster: VirtualScope, root: str) -> Optional[ScopeConfig]:
+    """Convert a virtual scope cluster to a ScopeConfig."""
+    if not cluster.files or not cluster.name:
+        return None
+
+    description = (
+        f"Virtual scope: {cluster.name} "
+        f"(spans {cluster.directories_spanned} modules, "
+        f"hub: {cluster.hub_file})"
+    )
+
+    dirs_spanned = set()
+    for f in cluster.files:
+        parts = Path(f).parts
+        if len(parts) > 1:
+            dirs_spanned.add(parts[0])
+
+    context = parse_context(
+        f"Cross-cutting concern detected from import graph.\n"
+        f"Hub file: {cluster.hub_file} "
+        f"(imported by {len(cluster.files) - 1} files across "
+        f"{cluster.directories_spanned} modules)\n"
+        f"\n"
+        f"Directories spanned: {', '.join(sorted(dirs_spanned))}\n"
+        f"Cohesion: {cluster.cohesion:.0%}"
+    )
+
+    full_paths = [os.path.join(root, f) for f in cluster.files]
+    token_est = estimate_scope_tokens(full_paths)
+
+    related = [f"{d}/.scope" for d in sorted(dirs_spanned)]
+
+    return ScopeConfig(
+        path=os.path.join(root, "virtual", f"{cluster.name}.scope"),
+        description=description,
+        includes=cluster.files,
+        excludes=[],
+        context=context,
+        related=related,
+        tags=["virtual", "cross-cutting", cluster.name.replace("_lifecycle", "")],
+        tokens_estimate=token_est,
+    )
+
+
+def format_virtual_scopes(scopes: List[ScopeConfig], root: str) -> str:
+    """Human-readable summary of detected virtual scopes."""
+    if not scopes:
+        return "No cross-cutting virtual scopes detected."
+
+    lines = [f"Detected {len(scopes)} virtual scope(s):", ""]
+    for scope in scopes:
+        lines.append(f"  {os.path.relpath(scope.path, root)}")
+        lines.append(f"    {scope.description}")
+        lines.append(f"    files: {len(scope.includes)}, ~{scope.tokens_estimate:,} tokens")
+        if scope.related:
+            lines.append(f"    related: {', '.join(scope.related)}")
+        lines.append("")
+
+    return "\n".join(lines)

dotscope/passes/voice.py

@@ -0,0 +1,162 @@
+"""Voice injection into resolve responses and canonical snippet extraction."""
+
+import ast
+import os
+from typing import Dict, List, Optional
+
+from ..models.intent import CanonicalExample
+
+
+def build_voice_response(
+    voice_config: dict,
+    root: str,
+    scope_files: List[str],
+    conventions: Optional[list] = None,
+) -> dict:
+    """Build the voice field for a resolve_scope response.
+
+    Returns a dict with mode, global rules, and optional convention voice.
+    """
+    result = {
+        "mode": voice_config.get("mode", "adaptive"),
+        "global": _serialize_global(voice_config),
+    }
+
+    # Convention-specific voice (if any file matches a convention with voice config)
+    if conventions:
+        for conv in conventions:
+            conv_voice = getattr(conv, "voice", None)
+            if not conv_voice:
+                # Check if convention dict has voice key
+                if isinstance(conv, dict):
+                    conv_voice = conv.get("voice")
+                else:
+                    continue
+            if not conv_voice:
+                continue
+
+            canonical = conv_voice.get("canonical_example") if isinstance(conv_voice, dict) else None
+            if canonical:
+                snippet = extract_canonical_snippet(canonical, root)
+                if snippet:
+                    result["convention"] = {
+                        "name": getattr(conv, "name", "") if not isinstance(conv, dict) else conv.get("name", ""),
+                        "style_notes": conv_voice.get("style_notes", "") if isinstance(conv_voice, dict) else "",
+                        "canonical_snippet": snippet,
+                    }
+                    break
+
+    return result
+
+
+def _serialize_global(voice_config: dict) -> str:
+    """Serialize voice rules as compact prose for the agent."""
+    rules = voice_config.get("rules", {})
+    if not rules:
+        return ""
+
+    parts = []
+    for key in ("typing", "docstrings", "error_handling", "structure", "density", "comments", "imports"):
+        val = rules.get(key)
+        if val:
+            parts.append(val.strip())
+
+    return " ".join(parts)
+
+
+def extract_canonical_snippet(
+    file_path: str,
+    repo_root: str,
+    max_lines: int = 40,
+) -> Optional[str]:
+    """Extract the first class or function as a canonical snippet.
+
+    Uses AST node locations to skip imports and module docstrings.
+    """
+    full_path = os.path.join(repo_root, file_path) if not os.path.isabs(file_path) else file_path
+    if not os.path.isfile(full_path):
+        return None
+
+    try:
+        with open(full_path, "r", encoding="utf-8") as f:
+            source = f.read()
+        tree = ast.parse(source)
+    except (SyntaxError, IOError, UnicodeDecodeError):
+        return None
+
+    # Find the first class or function definition
+    target = None
+    for node in ast.iter_child_nodes(tree):
+        if isinstance(node, ast.ClassDef):
+            target = node
+            break
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            target = node
+            break
+
+    if not target:
+        return None
+
+    # Extract source segment
+    snippet = ast.get_source_segment(source, target)
+    if not snippet:
+        # Fallback: extract by line numbers
+        lines = source.splitlines()
+        start = target.lineno - 1
+        end_line = getattr(target, "end_lineno", None) or (start + max_lines)
+        end = min(end_line, start + max_lines)
+        snippet = "\n".join(lines[start:end])
+
+    # Truncate if too long
+    snippet_lines = snippet.splitlines()
+    if len(snippet_lines) > max_lines:
+        snippet = "\n".join(snippet_lines[:max_lines]) + "\n    ..."
+
+    return snippet
+
+
+def select_canonical(
+    convention: object,
+    nodes: list,
+    history: Optional[dict],
+    repo_root: str,
+) -> Optional[CanonicalExample]:
+    """Pick the most representative file and extract its first class/function.
+
+    Selection: zero violations, most recently maintained, median length.
+    """
+    compliant = [n for n in nodes if not getattr(n, "violations", None)]
+    if not compliant:
+        return None
+
+    # Sort by recency if history available
+    if history and history.get("file_histories"):
+        compliant.sort(
+            key=lambda n: history["file_histories"]
+                .get(getattr(n, "file_path", ""), {})
+                .get("last_modified", ""),
+            reverse=True,
+        )
+
+    # Pick median length
+    lengths = []
+    for n in compliant[:10]:
+        fp = getattr(n, "file_path", "")
+        full = os.path.join(repo_root, fp)
+        try:
+            with open(full, "r", encoding="utf-8") as f:
+                length = len(f.readlines())
+        except (IOError, UnicodeDecodeError):
+            length = 0
+        lengths.append((n, length))
+
+    lengths.sort(key=lambda x: x[1])
+    best = lengths[len(lengths) // 2][0]
+    best_path = getattr(best, "file_path", "")
+
+    snippet = extract_canonical_snippet(best_path, repo_root)
+
+    return CanonicalExample(
+        file_path=best_path,
+        snippet=snippet,
+    )

dotscope/passes/voice_defaults.py

@@ -0,0 +1,28 @@
+"""Prescriptive voice defaults for new codebases.
+
+Applied when detect_codebase_maturity returns "new" (<10 files or
+<20 commits). Opinionated starting point that the developer can relax.
+"""
+
+from ..models.intent import DiscoveredVoice
+
+
+def prescriptive_defaults() -> DiscoveredVoice:
+    """Return strict voice config for a greenfield project."""
+    return DiscoveredVoice(
+        mode="prescriptive",
+        rules={
+            "typing": "Type hints on all function signatures. Return types always specified.",
+            "docstrings": "Google style. Imperative mood. One-line if the name explains it.",
+            "error_handling": "Domain exceptions. No bare excepts. Let unexpected errors propagate.",
+            "structure": "Early returns over nested conditionals. Guard clauses at the top.",
+            "density": "Concise. Comprehensions where readable. No filler variables.",
+            "comments": "Comments explain why, not what.",
+            "imports": "stdlib first, third-party second, local third. One import per line.",
+        },
+        stats={},
+        enforce={
+            "bare_excepts": "hold",
+            "missing_type_hints": "note",
+        },
+    )

dotscope/passes/voice_discovery.py

@@ -0,0 +1,245 @@
+"""Voice discovery: scan codebase for coding style patterns.
+
+Analyzes type hint adoption, docstring style, error handling,
+structural preferences, and comprehension density. On new codebases,
+returns prescriptive defaults. On existing codebases, codifies
+what's already there.
+"""
+
+import ast
+import os
+import re
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+from ..models.intent import DiscoveredVoice
+
+
+@dataclass
+class VoiceStats:
+    """Raw measurements from a codebase scan."""
+    total_functions: int = 0
+    typed_functions: int = 0
+    total_docstrings: int = 0
+    docstring_styles: Dict[str, int] = field(default_factory=lambda: {
+        "google": 0, "sphinx": 0, "numpy": 0, "other": 0,
+    })
+    total_excepts: int = 0
+    bare_excepts: int = 0
+    total_return_functions: int = 0
+    early_return_functions: int = 0
+    comprehensions: int = 0
+    for_loops: int = 0
+    files_analyzed: int = 0
+
+
+def detect_codebase_maturity(
+    ast_data: Dict[str, object],
+    history: Optional[object] = None,
+    override: Optional[str] = None,
+) -> str:
+    """Determine if this is a new or existing codebase.
+
+    Returns "new" or "existing".
+
+    Args:
+        override: "prescriptive" forces "new", "adaptive" forces "existing".
+    """
+    if override == "prescriptive":
+        return "new"
+    if override == "adaptive":
+        return "existing"
+
+    file_count = len(ast_data)
+    commit_count = getattr(history, "commits_analyzed", 0) if history else 0
+
+    if file_count < 10 or commit_count < 20:
+        return "new"
+    return "existing"
+
+
+def discover_voice(
+    ast_data: Dict[str, object],
+    repo_root: str,
+) -> DiscoveredVoice:
+    """Analyze the codebase to determine its existing voice.
+
+    Scans structural patterns across all files to determine type hint
+    adoption, docstring style, error handling, structural preferences,
+    and comprehension density.
+    """
+    stats = VoiceStats()
+
+    for path, analysis in ast_data.items():
+        full_path = os.path.join(repo_root, path)
+        if not os.path.isfile(full_path):
+            continue
+        if not path.endswith(".py"):
+            continue
+
+        # Count typed functions from existing FileAnalysis
+        for fn in getattr(analysis, "functions", []):
+            stats.total_functions += 1
+            if fn.return_type or any(
+                p for p in fn.params if ":" in str(p)
+            ):
+                stats.typed_functions += 1
+
+        # Re-parse for deeper analysis
+        try:
+            with open(full_path, "r", encoding="utf-8") as f:
+                source = f.read()
+            tree = ast.parse(source)
+        except (SyntaxError, IOError, UnicodeDecodeError):
+            continue
+
+        stats.files_analyzed += 1
+
+        # Docstrings
+        for node in ast.walk(tree):
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+                docstring = ast.get_docstring(node)
+                if docstring:
+                    stats.total_docstrings += 1
+                    style = _detect_docstring_style(docstring)
+                    stats.docstring_styles[style] += 1
+
+        # Exception handling
+        for node in ast.walk(tree):
+            if isinstance(node, ast.ExceptHandler):
+                stats.total_excepts += 1
+                if node.type is None:
+                    stats.bare_excepts += 1
+
+        # Early returns
+        for node in ast.walk(tree):
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                stats.total_return_functions += 1
+                if _has_early_return(node):
+                    stats.early_return_functions += 1
+
+        # Comprehensions vs loops
+        for node in ast.walk(tree):
+            if isinstance(node, (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp)):
+                stats.comprehensions += 1
+            elif isinstance(node, ast.For):
+                stats.for_loops += 1
+
+    return _synthesize_voice(stats)
+
+
+def _detect_docstring_style(docstring: str) -> str:
+    """Classify a docstring as Google, Sphinx, Numpy, or other."""
+    if re.search(r"^\s*(Args|Returns|Raises|Yields|Examples):", docstring, re.MULTILINE):
+        return "google"
+    if re.search(r"^\s*:(param|type|returns?|rtype|raises)\s*", docstring, re.MULTILINE):
+        return "sphinx"
+    if re.search(r"^\s*(Parameters|Returns|Raises)\s*\n\s*-{3,}", docstring, re.MULTILINE):
+        return "numpy"
+    return "other"
+
+
+def _has_early_return(node: ast.FunctionDef) -> bool:
+    """Check if a function has a return before its final statement."""
+    body = node.body
+    if len(body) <= 1:
+        return False
+    for stmt in body[:-1]:
+        if isinstance(stmt, ast.Return):
+            return True
+        if isinstance(stmt, ast.If):
+            for sub in ast.walk(stmt):
+                if isinstance(sub, ast.Return):
+                    return True
+    return False
+
+
+def _synthesize_voice(stats: VoiceStats) -> DiscoveredVoice:
+    """Convert raw stats into a voice description."""
+    rules = {}
+
+    # Type hints
+    hint_rate = stats.typed_functions / max(stats.total_functions, 1)
+    if hint_rate > 0.8:
+        rules["typing"] = "Strict type hints on all function signatures."
+    elif hint_rate > 0.4:
+        rules["typing"] = "Type hints used on most functions. Follow existing patterns."
+    else:
+        rules["typing"] = "Type hints encouraged on new code but not required."
+
+    # Docstrings
+    if stats.total_docstrings > 0:
+        dominant = max(stats.docstring_styles, key=stats.docstring_styles.get)
+        if dominant == "other":
+            rules["docstrings"] = "Minimal docstrings. Add only when behavior is non-obvious."
+        else:
+            rules["docstrings"] = f"{dominant.title()} style. Match existing docstrings."
+    else:
+        rules["docstrings"] = "Minimal docstrings. Add only when behavior is non-obvious."
+
+    # Error handling
+    bare_rate = stats.bare_excepts / max(stats.total_excepts, 1)
+    if bare_rate < 0.1:
+        rules["error_handling"] = "No bare excepts. Catch specific exception types."
+    elif bare_rate < 0.3:
+        rules["error_handling"] = "Avoid bare excepts in new code."
+    else:
+        rules["error_handling"] = "Match existing error handling patterns."
+
+    # Structure
+    early_rate = stats.early_return_functions / max(stats.total_return_functions, 1)
+    if early_rate > 0.6:
+        rules["structure"] = "Early returns preferred. Guard clauses at the top."
+    else:
+        rules["structure"] = "Match the pattern of the file being modified."
+
+    # Density
+    if stats.comprehensions > stats.for_loops * 0.5 and stats.comprehensions > 3:
+        rules["density"] = "Comprehensions preferred where readable."
+    else:
+        rules["density"] = "Explicit loops. Comprehensions for simple cases only."
+
+    enforce = compute_enforcement({
+        "type_hint_rate": round(hint_rate, 2),
+        "bare_except_rate": round(bare_rate, 2),
+    })
+
+    return DiscoveredVoice(
+        mode="adaptive",
+        rules=rules,
+        stats={
+            "type_hint_rate": round(hint_rate, 2),
+            "bare_except_rate": round(bare_rate, 2),
+            "early_return_rate": round(early_rate, 2),
+            "docstring_count": stats.total_docstrings,
+            "dominant_docstring_style": max(
+                stats.docstring_styles, key=stats.docstring_styles.get,
+            ) if stats.total_docstrings else None,
+            "files_analyzed": stats.files_analyzed,
+        },
+        enforce=enforce,
+    )
+
+
+def compute_enforcement(stats: dict) -> dict:
+    """Derive enforcement levels from actual codebase state.
+
+    Only enforce what the codebase already does.
+    """
+    enforce = {}
+
+    bare_rate = stats.get("bare_except_rate", 1.0)
+    if bare_rate < 0.10:
+        enforce["bare_excepts"] = "hold"
+    elif bare_rate < 0.30:
+        enforce["bare_excepts"] = "note"
+    else:
+        enforce["bare_excepts"] = False
+
+    hint_rate = stats.get("type_hint_rate", 0.0)
+    if hint_rate > 0.80:
+        enforce["missing_type_hints"] = "note"
+    else:
+        enforce["missing_type_hints"] = False
+
+    return enforce