codebase-intel 0.1.0__py3-none-any.whl

codebase_intel/core/types.py

@@ -0,0 +1,375 @@
+"""Core type definitions shared across all modules.
+
+Design principles:
+- Pydantic models for all structured data — never raw dicts
+- Immutable by default (frozen=True) — mutation must be explicit
+- Content-addressable where possible — xxhash for identity
+- Serializable to JSON for MCP transport and SQLite storage
+
+Edge cases addressed in type design:
+- File paths: always resolved to absolute, normalized (no symlink ambiguity)
+- Timestamps: always UTC, never naive datetimes
+- Identifiers: content-hash based where possible (stable across renames)
+- Line ranges: 1-indexed to match editor conventions, validated min<=max
+- Token counts: approximate — different models tokenize differently
+"""
+
+from __future__ import annotations
+
+import hashlib
+from datetime import UTC, datetime
+from enum import Enum
+from pathlib import Path
+from typing import Annotated, Any, Self
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+# ---------------------------------------------------------------------------
+# Enums
+# ---------------------------------------------------------------------------
+
+
+class NodeKind(str, Enum):
+    """Types of nodes in the semantic code graph."""
+
+    MODULE = "module"  # A file / compilation unit
+    CLASS = "class"
+    FUNCTION = "function"
+    METHOD = "method"
+    VARIABLE = "variable"  # Module-level constants, global state
+    INTERFACE = "interface"  # TS interfaces, Python Protocols
+    TYPE_ALIAS = "type_alias"
+    ENDPOINT = "endpoint"  # HTTP/gRPC/GraphQL endpoints
+    CONFIG = "config"  # Config files that affect behavior
+    UNKNOWN = "unknown"  # Parsed but unclassifiable
+
+
+class EdgeKind(str, Enum):
+    """Types of relationships between graph nodes."""
+
+    IMPORTS = "imports"  # Static import
+    DYNAMIC_IMPORT = "dynamic_import"  # importlib, dynamic require()
+    CALLS = "calls"  # Function/method invocation
+    INHERITS = "inherits"  # Class inheritance
+    IMPLEMENTS = "implements"  # Interface implementation
+    INSTANTIATES = "instantiates"  # Object creation
+    READS = "reads"  # Reads from variable/config
+    WRITES = "writes"  # Mutates variable/state
+    DEPENDS_ON = "depends_on"  # Generic dependency (package-level)
+    TESTS = "tests"  # Test file → source file relationship
+    CONFIGURES = "configures"  # Config file → module it affects
+    RE_EXPORTS = "re_exports"  # Barrel file re-exporting
+
+
+class Language(str, Enum):
+    """Supported source languages — 19 languages via tree-sitter-language-pack."""
+
+    PYTHON = "python"
+    JAVASCRIPT = "javascript"
+    TYPESCRIPT = "typescript"
+    TSX = "tsx"
+    GO = "go"
+    RUST = "rust"
+    JAVA = "java"
+    RUBY = "ruby"
+    C = "c"
+    CPP = "cpp"
+    CSHARP = "c_sharp"
+    PHP = "php"
+    SWIFT = "swift"
+    KOTLIN = "kotlin"
+    SCALA = "scala"
+    LUA = "lua"
+    DART = "dart"
+    ELIXIR = "elixir"
+    HASKELL = "haskell"
+    UNKNOWN = "unknown"  # tracked in graph but not parsed internally
+
+
+class DecisionStatus(str, Enum):
+    """Lifecycle of a decision record."""
+
+    DRAFT = "draft"  # Not yet finalized
+    ACTIVE = "active"  # Currently in effect
+    SUPERSEDED = "superseded"  # Replaced by another decision
+    DEPRECATED = "deprecated"  # Still in code but being removed
+    EXPIRED = "expired"  # Past its review date
+
+
+class ContractSeverity(str, Enum):
+    """How strictly a contract rule is enforced."""
+
+    ERROR = "error"  # Must not violate — blocks generation
+    WARNING = "warning"  # Should not violate — agent sees it as guidance
+    INFO = "info"  # Nice to know — lowest priority context
+
+
+class DriftLevel(str, Enum):
+    """Severity of detected drift."""
+
+    NONE = "none"
+    LOW = "low"  # Minor: line numbers shifted
+    MEDIUM = "medium"  # Moderate: content changed but structure intact
+    HIGH = "high"  # Major: code significantly different
+    CRITICAL = "critical"  # Anchor deleted or file removed
+
+
+# ---------------------------------------------------------------------------
+# Value objects (immutable, content-addressed)
+# ---------------------------------------------------------------------------
+
+
+class FileFingerprint(BaseModel):
+    """Content-addressable identity for a file at a point in time.
+
+    Edge cases:
+    - Renamed files: same content hash → detectable as rename, not delete+create
+    - Binary files: we hash them but don't parse internals
+    - Empty files: valid fingerprint (empty string hashes consistently)
+    - Encoding issues: we read as bytes for hashing, decode for parsing separately
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    path: Path
+    content_hash: str = Field(description="xxhash of file bytes")
+    size_bytes: int = Field(ge=0)
+    last_modified: datetime
+    language: Language = Language.UNKNOWN
+
+    @field_validator("last_modified")
+    @classmethod
+    def ensure_utc(cls, v: datetime) -> datetime:
+        if v.tzinfo is None:
+            return v.replace(tzinfo=UTC)
+        return v.astimezone(UTC)
+
+    @field_validator("path")
+    @classmethod
+    def normalize_path(cls, v: Path) -> Path:
+        return v.resolve()
+
+
+class LineRange(BaseModel):
+    """A range of lines in a file (1-indexed, inclusive).
+
+    Edge cases:
+    - Single-line range: start == end (valid)
+    - Full-file range: start=1, end=line_count
+    - After refactor: line numbers shift — we re-anchor via content hash
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    start: int = Field(ge=1)
+    end: int = Field(ge=1)
+
+    @model_validator(mode="after")
+    def start_before_end(self) -> Self:
+        if self.start > self.end:
+            msg = f"Line range start ({self.start}) must be <= end ({self.end})"
+            raise ValueError(msg)
+        return self
+
+    @property
+    def span(self) -> int:
+        return self.end - self.start + 1
+
+
+class CodeAnchor(BaseModel):
+    """Links a non-code artifact (decision, contract) to a specific code location.
+
+    Edge cases:
+    - File renamed: content_hash lets us find it at new path
+    - Lines shifted: content_hash of the anchored region lets us re-locate
+    - File deleted: anchor becomes orphaned — drift detector flags it
+    - Function renamed: symbol_name helps re-anchor even if hash changes
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    file_path: Path
+    line_range: LineRange | None = None
+    symbol_name: str | None = None  # e.g., "RateLimiter.check"
+    content_hash: str | None = None  # hash of the anchored code region
+
+    @field_validator("file_path")
+    @classmethod
+    def normalize_path(cls, v: Path) -> Path:
+        return v.resolve()
+
+    def is_orphaned(self, existing_paths: set[Path]) -> bool:
+        """Check if the anchor's file still exists."""
+        return self.file_path.resolve() not in existing_paths
+
+
+class TokenBudget(BaseModel):
+    """Token budget for context assembly.
+
+    Edge cases:
+    - Zero budget: return empty context with metadata only
+    - Budget smaller than minimum useful context: return highest-priority items
+      with a truncation warning
+    - Model-specific tokenization: budget is approximate since different models
+      tokenize differently. We use tiktoken cl100k_base as a reasonable baseline
+      and include a safety margin.
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    total: int = Field(gt=0, description="Total tokens available for context")
+    reserved_for_response: int = Field(
+        default=0,
+        ge=0,
+        description="Tokens to reserve for the agent's response",
+    )
+    safety_margin_pct: float = Field(
+        default=0.1,
+        ge=0,
+        le=0.5,
+        description="Percentage to hold back for tokenization variance",
+    )
+
+    @property
+    def usable(self) -> int:
+        """Actual tokens available for context after reserves."""
+        raw = self.total - self.reserved_for_response
+        margin = int(raw * self.safety_margin_pct)
+        return max(0, raw - margin)
+
+
+# ---------------------------------------------------------------------------
+# Graph node & edge
+# ---------------------------------------------------------------------------
+
+
+class GraphNode(BaseModel):
+    """A node in the semantic code graph.
+
+    Edge cases:
+    - Generated code: marked with is_generated=True, lower trust weight
+    - Vendored/third-party: marked with is_external=True, read-only context
+    - Test files: marked with is_test=True, linked to source via TESTS edge
+    - Barrel/index files: may have many RE_EXPORTS edges, low own content
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    node_id: str = Field(description="Stable ID: hash of (path, kind, name)")
+    kind: NodeKind
+    name: str
+    qualified_name: str = Field(
+        description="Full path: module.Class.method"
+    )
+    file_path: Path
+    line_range: LineRange | None = None
+    language: Language = Language.UNKNOWN
+    content_hash: str | None = None
+    docstring: str | None = None
+
+    is_generated: bool = False
+    is_external: bool = False
+    is_test: bool = False
+    is_entry_point: bool = False
+
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    @staticmethod
+    def make_id(file_path: Path, kind: NodeKind, name: str) -> str:
+        """Deterministic node ID from its identity components."""
+        raw = f"{file_path.resolve()}:{kind.value}:{name}"
+        return hashlib.sha256(raw.encode()).hexdigest()[:16]
+
+
+class GraphEdge(BaseModel):
+    """A directed edge between two graph nodes.
+
+    Edge cases:
+    - Conditional edges: import inside if TYPE_CHECKING — marked is_type_only
+    - Dynamic edges: importlib.import_module() — marked with confidence < 1.0
+    - Circular edges: A → B → A — valid, traversal must handle
+    - Cross-language edges: Python calling C extension — low confidence
+    """
+
+    model_config = ConfigDict(frozen=True)
+
+    source_id: str
+    target_id: str
+    kind: EdgeKind
+    confidence: float = Field(
+        default=1.0,
+        ge=0.0,
+        le=1.0,
+        description="1.0 = static import, <1.0 = inferred/dynamic",
+    )
+    is_type_only: bool = Field(
+        default=False,
+        description="True for TYPE_CHECKING imports, type annotations only",
+    )
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+
+# ---------------------------------------------------------------------------
+# Context assembly types
+# ---------------------------------------------------------------------------
+
+
+class ContextPriority(str, Enum):
+    """Priority levels for context items during budget allocation."""
+
+    CRITICAL = "critical"  # Must include — directly referenced files
+    HIGH = "high"  # Should include — immediate dependencies, active decisions
+    MEDIUM = "medium"  # Include if budget allows — transitive deps, older decisions
+    LOW = "low"  # Nice to have — tangential context, info-level contracts
+
+
+class ContextItem(BaseModel):
+    """A single item in the assembled context payload.
+
+    This is the universal wrapper — every piece of context (file content,
+    decision record, contract rule) gets wrapped in this for the orchestrator
+    to prioritize and budget.
+    """
+
+    source: str = Field(description="Which module provided this: graph|decisions|contracts")
+    item_type: str = Field(description="Specific type: file_content|decision|contract_rule|warning")
+    priority: ContextPriority
+    estimated_tokens: int = Field(ge=0)
+    content: str
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    freshness_score: float = Field(
+        default=1.0,
+        ge=0.0,
+        le=1.0,
+        description="1.0 = just validated, 0.0 = very stale",
+    )
+
+
+class AssembledContext(BaseModel):
+    """The final context payload sent to an AI agent.
+
+    Edge cases:
+    - Empty context: valid if the task has no relevant files (new greenfield code)
+    - Truncated context: items were dropped due to budget — truncated=True with
+      a summary of what was dropped
+    - Conflicting context: contains contradictions — conflicts list populated
+    - Partial context: some modules unavailable — warnings list populated
+    """
+
+    items: list[ContextItem] = Field(default_factory=list)
+    total_tokens: int = 0
+    budget_tokens: int = 0
+    truncated: bool = False
+    dropped_count: int = Field(
+        default=0,
+        description="Number of items dropped due to budget",
+    )
+    conflicts: list[str] = Field(
+        default_factory=list,
+        description="Human-readable conflict descriptions",
+    )
+    warnings: list[str] = Field(
+        default_factory=list,
+        description="Degradation notices (partial init, stale data, etc.)",
+    )
+    assembly_time_ms: float = 0.0

codebase_intel/decisions/__init__.py

@@ -0,0 +1 @@
+"""Decision Journal — structured records of architectural and business decisions."""

codebase_intel/decisions/miner.py

@@ -0,0 +1,297 @@
+"""Git history miner — extracts decision candidates from PRs, commits, and comments.
+
+The goal is to auto-discover decisions that were made but never formally recorded.
+Developers make decisions constantly in PR descriptions, commit messages, and code
+review comments — this module surfaces them.
+
+This is SUGGESTION-based, not automatic. Every mined decision has confidence < 1.0
+and should be reviewed by a human before becoming official.
+
+Edge cases:
+- PR description is empty: skip (nothing to mine)
+- PR description is a template with checkboxes: extract only non-template content
+- Commit message is "fix" or "wip": skip (no decision content)
+- Multiple decisions in one PR: extract each as a separate candidate
+- Decision language in non-English: basic support via keyword matching only
+- Merge commits: skip (they reference the PR, not new decisions)
+- Squash commits: contain the full PR description — high-value target
+- Revert commits: flag the original decision as potentially superseded
+- Git history is very large: limit mining depth with max_commits parameter
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from codebase_intel.core.types import CodeAnchor, DecisionStatus
+from codebase_intel.decisions.models import DecisionRecord
+
+if TYPE_CHECKING:
+    from codebase_intel.core.config import DecisionConfig
+
+logger = logging.getLogger(__name__)
+
+# Keywords that suggest a commit/PR contains a decision
+DECISION_KEYWORDS = [
+    # Architecture
+    "decided", "decision", "chose", "chosen", "opted for", "switched to",
+    "migrated from", "replaced", "instead of", "rather than",
+    # Reasoning
+    "because", "reason:", "rationale:", "why:", "trade-off", "tradeoff",
+    "considered", "evaluated", "compared",
+    # Constraints
+    "compliance", "regulation", "requirement", "sla", "must not", "cannot",
+    "forbidden", "prohibited",
+    # Breaking changes
+    "breaking change", "breaking:", "deprecated", "removed",
+    # Architecture-specific
+    "adr", "architecture decision", "design decision", "rfc",
+]
+
+# Patterns that indicate a commit should be skipped
+SKIP_PATTERNS = [
+    r"^merge\s",
+    r"^wip\b",
+    r"^fix\s*(typo|lint|format|style)",
+    r"^chore\s*:",
+    r"^bump\s+version",
+    r"^update\s+lock",
+    r"^auto-generated",
+]
+
+
+@dataclass
+class DecisionCandidate:
+    """A potential decision extracted from git history, pending human review."""
+
+    title: str
+    context: str
+    decision_text: str
+    source_type: str  # "commit", "pr_description", "pr_comment"
+    source_ref: str  # commit hash, PR URL
+    author: str
+    created_at: datetime
+    changed_files: list[Path] = field(default_factory=list)
+    confidence: float = 0.5
+    keywords_matched: list[str] = field(default_factory=list)
+
+    def to_decision_record(self, decision_id: str) -> DecisionRecord:
+        """Convert this candidate to a draft DecisionRecord."""
+        anchors = [
+            CodeAnchor(file_path=fp) for fp in self.changed_files[:10]
+        ]
+
+        return DecisionRecord(
+            id=decision_id,
+            title=self.title,
+            status=DecisionStatus.DRAFT,
+            context=self.context,
+            decision=self.decision_text,
+            code_anchors=anchors,
+            created_at=self.created_at,
+            author=self.author,
+            source="git-mined",
+            source_ref=self.source_ref,
+            confidence=self.confidence,
+            tags=["auto-mined"],
+        )
+
+
+class GitMiner:
+    """Mines git history for decision candidates."""
+
+    def __init__(
+        self,
+        config: DecisionConfig,
+        project_root: Path,
+    ) -> None:
+        self._config = config
+        self._project_root = project_root
+
+    async def mine_commits(
+        self,
+        max_commits: int = 500,
+        since_days: int = 90,
+    ) -> list[DecisionCandidate]:
+        """Mine recent commit messages for decision candidates.
+
+        Edge cases:
+        - Not a git repo: return empty list with warning
+        - Shallow clone: limited history available — mine what we have
+        - Binary commits (large media files): skip based on file extensions
+        - Encoding issues in commit messages: handle gracefully
+        """
+        try:
+            from git import Repo
+        except ImportError:
+            logger.warning("GitPython not installed — cannot mine commits")
+            return []
+
+        try:
+            repo = Repo(self._project_root, search_parent_directories=True)
+        except Exception:
+            logger.warning("Not a git repository: %s", self._project_root)
+            return []
+
+        candidates: list[DecisionCandidate] = []
+        count = 0
+
+        since_dt = datetime.now(UTC).replace(
+            day=max(1, datetime.now(UTC).day),
+        )
+
+        for commit in repo.iter_commits(max_count=max_commits):
+            count += 1
+
+            message = commit.message.strip()
+            if not message:
+                continue
+
+            # Skip noise commits
+            if self._should_skip(message):
+                continue
+
+            # Check for decision keywords
+            matched_keywords = self._match_keywords(message)
+            if not matched_keywords:
+                continue
+
+            # Extract changed files
+            changed_files: list[Path] = []
+            try:
+                if commit.parents:
+                    diff = commit.parents[0].diff(commit)
+                    changed_files = [
+                        self._project_root / d.a_path
+                        for d in diff
+                        if d.a_path
+                    ]
+            except Exception:
+                pass  # Diff extraction is best-effort
+
+            # Build candidate
+            title = self._extract_title(message)
+            context, decision_text = self._extract_context_and_decision(message)
+
+            confidence = self._compute_confidence(
+                message, matched_keywords, changed_files
+            )
+
+            candidates.append(DecisionCandidate(
+                title=title,
+                context=context,
+                decision_text=decision_text,
+                source_type="commit",
+                source_ref=str(commit.hexsha)[:12],
+                author=commit.author.name if commit.author else "unknown",
+                created_at=datetime.fromtimestamp(commit.committed_date, tz=UTC),
+                changed_files=changed_files[:20],
+                confidence=confidence,
+                keywords_matched=matched_keywords,
+            ))
+
+        logger.info(
+            "Mined %d commits, found %d decision candidates",
+            count,
+            len(candidates),
+        )
+
+        return candidates
+
+    def _should_skip(self, message: str) -> bool:
+        """Check if a commit message should be skipped.
+
+        Edge case: multi-line messages — check only the first line for
+        skip patterns but check all lines for decision keywords.
+        """
+        first_line = message.split("\n")[0].lower().strip()
+        return any(re.match(pattern, first_line) for pattern in SKIP_PATTERNS)
+
+    def _match_keywords(self, message: str) -> list[str]:
+        """Find decision-indicating keywords in the message."""
+        message_lower = message.lower()
+        return [kw for kw in DECISION_KEYWORDS if kw in message_lower]
+
+    def _extract_title(self, message: str) -> str:
+        """Extract a title from the commit message.
+
+        Convention: first line of the commit message, truncated at 80 chars.
+        Edge case: first line is very long (someone put everything on one line).
+        """
+        first_line = message.split("\n")[0].strip()
+        if len(first_line) > 80:
+            return first_line[:77] + "..."
+        return first_line
+
+    def _extract_context_and_decision(
+        self, message: str
+    ) -> tuple[str, str]:
+        """Separate the context ("why") from the decision ("what") in a message.
+
+        Heuristic: lines before "because"/"reason:" are the decision,
+        lines after are the context. If no such separator, the whole
+        message is both context and decision.
+
+        Edge case: message with no clear separation — use the first line
+        as the decision and the rest as context.
+        """
+        lines = message.strip().split("\n")
+
+        if len(lines) <= 1:
+            return message, message
+
+        first_line = lines[0].strip()
+        body = "\n".join(lines[1:]).strip()
+
+        if not body:
+            return first_line, first_line
+
+        return body, first_line
+
+    def _compute_confidence(
+        self,
+        message: str,
+        keywords: list[str],
+        changed_files: list[Path],
+    ) -> float:
+        """Compute confidence score for a mined decision candidate.
+
+        Factors:
+        - Number of keywords matched (more = higher)
+        - Message length (longer = more context = higher)
+        - Number of files changed (fewer = more focused = higher)
+        - Presence of reasoning words (because, reason) = higher
+        - Presence of "adr" or "decision" = much higher
+
+        Score range: 0.2 (barely qualifying) to 0.8 (strong signal).
+        Never 1.0 — that requires human confirmation.
+        """
+        score = 0.3  # Base score for matching any keyword
+
+        # Keyword count bonus
+        score += min(0.2, len(keywords) * 0.05)
+
+        # Message length bonus (meaningful messages are longer)
+        if len(message) > 200:
+            score += 0.1
+        if len(message) > 500:
+            score += 0.1
+
+        # Focused changes (fewer files = clearer decision)
+        if 1 <= len(changed_files) <= 5:
+            score += 0.05
+
+        # High-signal keywords
+        message_lower = message.lower()
+        if any(kw in message_lower for kw in ("adr", "architecture decision", "design decision")):
+            score += 0.15
+
+        if any(kw in message_lower for kw in ("because", "reason:", "rationale:")):
+            score += 0.1
+
+        return min(0.8, score)