emdash-core 0.1.7__py3-none-any.whl

emdash_core/ingestion/change_detector.py

@@ -0,0 +1,150 @@
+"""Change detection for incremental indexing using git diff."""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+import git
+
+from ..utils.logger import log
+
+
+@dataclass
+class ChangedFiles:
+    """Files changed since last indexing."""
+
+    added: list[Path] = field(default_factory=list)
+    modified: list[Path] = field(default_factory=list)
+    deleted: list[Path] = field(default_factory=list)
+
+    @property
+    def all_to_index(self) -> list[Path]:
+        """Get all files that need to be (re)indexed."""
+        return self.added + self.modified
+
+    @property
+    def total_changes(self) -> int:
+        """Total number of changed files."""
+        return len(self.added) + len(self.modified) + len(self.deleted)
+
+    def __bool__(self) -> bool:
+        """True if there are any changes."""
+        return self.total_changes > 0
+
+
+class ChangeDetector:
+    """Detects files changed since last indexing using git diff."""
+
+    def __init__(self, repo: git.Repo, last_indexed_commit: Optional[str] = None):
+        """Initialize change detector.
+
+        Args:
+            repo: Git repository
+            last_indexed_commit: SHA of commit at last index, or None for full index
+        """
+        self.repo = repo
+        self.last_indexed_commit = last_indexed_commit
+        self.repo_root = Path(repo.working_dir)
+
+    def get_current_commit(self) -> str:
+        """Get current HEAD commit SHA."""
+        return self.repo.head.commit.hexsha
+
+    def get_changed_files(self, extensions: list[str] = None) -> ChangedFiles:
+        """Find files changed since last index.
+
+        Uses git diff to detect:
+        - Added files (A)
+        - Modified files (M)
+        - Deleted files (D)
+        - Renamed files (R) - treated as delete + add
+
+        Args:
+            extensions: Optional list of extensions to filter (e.g., ['.py', '.ts'])
+
+        Returns:
+            ChangedFiles with categorized changes
+        """
+        if not self.last_indexed_commit:
+            log.info("No previous index commit - full index required")
+            return ChangedFiles()
+
+        try:
+            # Verify the commit exists
+            try:
+                old_commit = self.repo.commit(self.last_indexed_commit)
+            except git.BadName:
+                log.warning(f"Previous commit {self.last_indexed_commit[:8]} not found - full index required")
+                return ChangedFiles()
+
+            current_commit = self.repo.head.commit
+
+            # Get diff between last indexed commit and current HEAD
+            diff = old_commit.diff(current_commit)
+
+            added = []
+            modified = []
+            deleted = []
+
+            for change in diff:
+                # Handle different change types
+                if change.change_type == 'A':  # Added
+                    file_path = self.repo_root / change.b_path
+                    if self._should_include(file_path, extensions):
+                        added.append(file_path)
+
+                elif change.change_type == 'M':  # Modified
+                    file_path = self.repo_root / change.b_path
+                    if self._should_include(file_path, extensions):
+                        modified.append(file_path)
+
+                elif change.change_type == 'D':  # Deleted
+                    file_path = self.repo_root / change.a_path
+                    if self._should_include(file_path, extensions):
+                        deleted.append(file_path)
+
+                elif change.change_type == 'R':  # Renamed
+                    # Treat as delete old + add new
+                    old_path = self.repo_root / change.a_path
+                    new_path = self.repo_root / change.b_path
+                    if self._should_include(old_path, extensions):
+                        deleted.append(old_path)
+                    if self._should_include(new_path, extensions):
+                        added.append(new_path)
+
+                elif change.change_type in ('C', 'T'):  # Copied or Type changed
+                    file_path = self.repo_root / change.b_path
+                    if self._should_include(file_path, extensions):
+                        modified.append(file_path)
+
+            result = ChangedFiles(added=added, modified=modified, deleted=deleted)
+
+            log.info(f"Detected changes: {len(added)} added, {len(modified)} modified, {len(deleted)} deleted")
+
+            return result
+
+        except Exception as e:
+            log.error(f"Error detecting changes: {e}")
+            return ChangedFiles()
+
+    def _should_include(self, file_path: Path, extensions: Optional[list[str]]) -> bool:
+        """Check if file should be included based on extension filter.
+
+        Args:
+            file_path: Path to file
+            extensions: Optional list of extensions to include
+
+        Returns:
+            True if file should be included
+        """
+        if extensions is None:
+            return True
+        return file_path.suffix.lower() in [ext.lower() for ext in extensions]
+
+    def has_uncommitted_changes(self) -> bool:
+        """Check if there are uncommitted changes in the working directory.
+
+        Returns:
+            True if there are uncommitted changes
+        """
+        return self.repo.is_dirty(untracked_files=True)

emdash_core/ingestion/git/__init__.py

@@ -0,0 +1,5 @@
+"""Git analysis module for EmDash."""
+
+from .commit_analyzer import CommitAnalyzer
+
+__all__ = ["CommitAnalyzer"]

emdash_core/ingestion/git/commit_analyzer.py

@@ -0,0 +1,196 @@
+"""Analyze Git commit history."""
+
+from collections import defaultdict
+from datetime import datetime
+from pathlib import Path
+from typing import List, Dict
+
+from git import Repo
+
+from ...core.models import (
+    CommitEntity,
+    AuthorEntity,
+    FileModification,
+    GitData,
+    RepositoryEntity,
+)
+from ...utils.logger import log
+
+
+class CommitAnalyzer:
+    """Analyzes Git commit history and extracts metadata."""
+
+    def __init__(self, repo: Repo, max_commits: int = None):
+        """Initialize commit analyzer.
+
+        Args:
+            repo: Git repository
+            max_commits: Maximum number of commits to analyze (None = all)
+        """
+        self.repo = repo
+        self.max_commits = max_commits
+
+    def analyze(self, repo_entity: RepositoryEntity) -> GitData:
+        """Analyze commit history and extract Git data.
+
+        Args:
+            repo_entity: Repository entity
+
+        Returns:
+            GitData containing commits, authors, and modifications
+        """
+        log.info("Analyzing Git commit history...")
+
+        commits = []
+        modifications = []
+        author_stats = defaultdict(lambda: {
+            'name': '',
+            'email': '',
+            'commits': 0,
+            'lines_added': 0,
+            'lines_deleted': 0,
+            'first_commit': None,
+            'last_commit': None,
+        })
+
+        # Iterate through commits
+        commit_count = 0
+        for commit in self.repo.iter_commits():
+            if self.max_commits and commit_count >= self.max_commits:
+                break
+
+            commit_entity = self._extract_commit(commit)
+            commits.append(commit_entity)
+
+            # Extract file modifications
+            file_mods = self._extract_modifications(commit)
+            modifications.extend(file_mods)
+
+            # Update author statistics
+            email = commit.author.email
+            author_stats[email]['name'] = commit.author.name
+            author_stats[email]['email'] = email
+            author_stats[email]['commits'] += 1
+            author_stats[email]['lines_added'] += commit.stats.total['insertions']
+            author_stats[email]['lines_deleted'] += commit.stats.total['deletions']
+
+            timestamp = datetime.fromtimestamp(commit.committed_date)
+            if author_stats[email]['first_commit'] is None:
+                author_stats[email]['first_commit'] = timestamp
+            author_stats[email]['last_commit'] = timestamp
+
+            commit_count += 1
+
+        # Create author entities
+        authors = [
+            AuthorEntity(
+                email=stats['email'],
+                name=stats['name'],
+                first_commit=stats['first_commit'],
+                last_commit=stats['last_commit'],
+                total_commits=stats['commits'],
+                total_lines_added=stats['lines_added'],
+                total_lines_deleted=stats['lines_deleted'],
+            )
+            for stats in author_stats.values()
+        ]
+
+        log.info(f"Analyzed {len(commits)} commits, {len(authors)} authors")
+
+        return GitData(
+            repository=repo_entity,
+            commits=commits,
+            modifications=modifications,
+            authors=authors,
+        )
+
+    def _extract_commit(self, commit) -> CommitEntity:
+        """Extract a commit entity from a git commit.
+
+        Args:
+            commit: GitPython commit object
+
+        Returns:
+            CommitEntity
+        """
+        parent_shas = [parent.hexsha for parent in commit.parents]
+
+        return CommitEntity(
+            sha=commit.hexsha,
+            message=commit.message,
+            timestamp=datetime.fromtimestamp(commit.committed_date),
+            author_name=commit.author.name,
+            author_email=commit.author.email,
+            committer_name=commit.committer.name,
+            committer_email=commit.committer.email,
+            insertions=commit.stats.total['insertions'],
+            deletions=commit.stats.total['deletions'],
+            files_changed=commit.stats.total['files'],
+            is_merge=len(parent_shas) > 1,
+            parent_shas=parent_shas,
+        )
+
+    def _extract_modifications(self, commit) -> List[FileModification]:
+        """Extract file modifications from a commit.
+
+        Args:
+            commit: GitPython commit object
+
+        Returns:
+            List of FileModification objects
+        """
+        modifications = []
+
+        try:
+            # Get diffs from parent (if exists)
+            if commit.parents:
+                diffs = commit.parents[0].diff(commit, create_patch=True)
+            else:
+                # First commit - all files are new
+                diffs = commit.diff(None, create_patch=True)
+
+            for diff in diffs:
+                # Determine change type
+                if diff.new_file:
+                    change_type = "added"
+                elif diff.deleted_file:
+                    change_type = "deleted"
+                elif diff.renamed_file:
+                    change_type = "renamed"
+                else:
+                    change_type = "modified"
+
+                # Get file path (handle renames)
+                relative_path = diff.b_path if diff.b_path else diff.a_path
+                old_relative_path = diff.a_path if diff.renamed_file else None
+
+                # Convert relative paths to absolute paths (matching File nodes)
+                repo_root = Path(self.repo.working_dir)
+                file_path = str(repo_root / relative_path) if relative_path else None
+                old_path = str(repo_root / old_relative_path) if old_relative_path else None
+
+                # Calculate insertions/deletions (rough estimate from diff stats)
+                insertions = 0
+                deletions = 0
+
+                if hasattr(diff, 'diff') and diff.diff:
+                    diff_text = diff.diff.decode('utf-8', errors='ignore')
+                    for line in diff_text.split('\n'):
+                        if line.startswith('+') and not line.startswith('+++'):
+                            insertions += 1
+                        elif line.startswith('-') and not line.startswith('---'):
+                            deletions += 1
+
+                modifications.append(FileModification(
+                    commit_sha=commit.hexsha,
+                    file_path=file_path,
+                    change_type=change_type,
+                    insertions=insertions,
+                    deletions=deletions,
+                    old_path=old_path,
+                ))
+
+        except Exception as e:
+            log.warning(f"Failed to extract modifications for commit {commit.hexsha}: {e}")
+
+        return modifications

emdash_core/ingestion/github/__init__.py

@@ -0,0 +1,6 @@
+"""GitHub integration for EmDash."""
+
+from .pr_fetcher import PRFetcher
+from .task_extractor import TaskExtractor
+
+__all__ = ["PRFetcher", "TaskExtractor"]

emdash_core/ingestion/github/pr_fetcher.py

@@ -0,0 +1,296 @@
+"""GitHub Pull Request fetcher using gh CLI."""
+
+import json
+import os
+import re
+import shutil
+import subprocess
+from datetime import datetime
+from typing import Optional
+
+from ...core.models import PullRequestEntity
+from ...utils.logger import log
+
+
+class PRFetcher:
+    """Fetches pull requests using gh CLI (supports private repos)."""
+
+    # Pattern to extract owner/repo from GitHub URLs
+    GITHUB_URL_PATTERN = re.compile(
+        r"(?:https?://)?(?:www\.)?github\.com[/:]([^/]+)/([^/.]+)(?:\.git)?/?$"
+    )
+
+    def __init__(
+        self,
+        owner: str,
+        repo: str,
+        token: Optional[str] = None,
+    ):
+        """Initialize PR fetcher.
+
+        Args:
+            owner: GitHub repository owner
+            repo: GitHub repository name
+            token: GitHub personal access token (ignored, uses gh CLI auth)
+        """
+        self.owner = owner
+        self.repo = repo
+        self.repo_path = f"{owner}/{repo}"
+
+        # Find gh CLI binary
+        self.gh_path = self._find_gh_cli()
+        if not self.gh_path:
+            log.warning(
+                "gh CLI not found. Install with 'brew install gh' and authenticate with 'gh auth login'. "
+                "PR fetching will be skipped."
+            )
+
+    def _find_gh_cli(self) -> Optional[str]:
+        """Find the gh CLI binary path."""
+        # Check common locations
+        paths_to_check = [
+            shutil.which("gh"),
+            "/opt/homebrew/bin/gh",
+            "/usr/local/bin/gh",
+            "/usr/bin/gh",
+        ]
+        for path in paths_to_check:
+            if path and shutil.os.path.isfile(path):
+                return path
+        return None
+
+    def _run_gh(self, args: list[str]) -> Optional[str]:
+        """Run a gh CLI command and return output."""
+        if not self.gh_path:
+            return None
+        try:
+            # Copy environment but remove GITHUB_TOKEN to let gh use its own OAuth
+            env = os.environ.copy()
+            env.pop("GITHUB_TOKEN", None)
+            env.pop("GH_TOKEN", None)
+
+            result = subprocess.run(
+                [self.gh_path] + args,
+                capture_output=True,
+                text=True,
+                timeout=120,
+                env=env,
+            )
+            if result.returncode == 0:
+                return result.stdout
+            else:
+                log.error(f"gh CLI error: {result.stderr}")
+                return None
+        except subprocess.TimeoutExpired:
+            log.error("gh CLI command timed out")
+            return None
+        except Exception as e:
+            log.error(f"gh CLI error: {e}")
+            return None
+
+    @classmethod
+    def extract_repo_info(cls, remote_url: str) -> tuple[Optional[str], Optional[str]]:
+        """Extract owner and repo name from a GitHub remote URL.
+
+        Args:
+            remote_url: Git remote URL (HTTPS or SSH format)
+
+        Returns:
+            Tuple of (owner, repo) or (None, None) if not a GitHub URL
+
+        Examples:
+            >>> PRFetcher.extract_repo_info("https://github.com/owner/repo.git")
+            ('owner', 'repo')
+            >>> PRFetcher.extract_repo_info("git@github.com:owner/repo.git")
+            ('owner', 'repo')
+        """
+        # Handle SSH format: git@github.com:owner/repo.git
+        if remote_url.startswith("git@github.com:"):
+            parts = remote_url.replace("git@github.com:", "").replace(".git", "").split("/")
+            if len(parts) >= 2:
+                return parts[0], parts[1]
+
+        # Handle HTTPS format
+        match = cls.GITHUB_URL_PATTERN.match(remote_url)
+        if match:
+            return match.group(1), match.group(2)
+
+        return None, None
+
+    def fetch_prs(
+        self,
+        state: str = "all",
+        limit: Optional[int] = 100,
+        since: Optional[datetime] = None,
+    ) -> list[PullRequestEntity]:
+        """Fetch pull requests from the repository using gh CLI.
+
+        Args:
+            state: PR state filter ("open", "closed", "all")
+            limit: Maximum number of PRs to fetch (None for all)
+            since: Only fetch PRs updated after this datetime
+
+        Returns:
+            List of PullRequestEntity objects
+        """
+        if not self.gh_path:
+            log.error("gh CLI not available. Cannot fetch PRs.")
+            return []
+
+        prs = []
+
+        # Map state to gh CLI format
+        states_to_fetch = []
+        if state == "all":
+            states_to_fetch = ["open", "closed", "merged"]
+        elif state == "closed":
+            states_to_fetch = ["closed", "merged"]
+        else:
+            states_to_fetch = [state]
+
+        for pr_state in states_to_fetch:
+            # Build gh pr list command - request minimal fields to avoid GraphQL limits
+            args = [
+                "pr", "list",
+                "-R", self.repo_path,
+                "--state", pr_state,
+                "--limit", str(limit or 100),
+                "--json", "number,title,body,state,createdAt,author,mergedAt,labels,additions,deletions,baseRefName,headRefName"
+            ]
+
+            output = self._run_gh(args)
+            if not output:
+                continue
+
+            try:
+                pr_list = json.loads(output)
+            except json.JSONDecodeError as e:
+                log.error(f"Failed to parse gh output: {e}")
+                continue
+
+            for pr_data in pr_list:
+                # Check date filter
+                if since:
+                    created_at = self._parse_datetime(pr_data.get("createdAt"))
+                    if created_at and created_at < since:
+                        continue
+
+                pr_entity = self._extract_pr_from_json(pr_data)
+                prs.append(pr_entity)
+
+                if len(prs) % 10 == 0:
+                    log.info(f"Fetched {len(prs)} PRs...")
+
+                # Check limit
+                if limit and len(prs) >= limit:
+                    break
+
+            if limit and len(prs) >= limit:
+                break
+
+        log.info(f"Fetched {len(prs)} pull requests from {self.repo_path}")
+        return prs
+
+    def fetch_pr_files(self, pr_number: int) -> list[str]:
+        """Get files changed in a specific PR.
+
+        Args:
+            pr_number: Pull request number
+
+        Returns:
+            List of file paths that were modified
+        """
+        if not self.gh_path:
+            return []
+
+        args = [
+            "pr", "view", str(pr_number),
+            "-R", self.repo_path,
+            "--json", "files"
+        ]
+
+        output = self._run_gh(args)
+        if not output:
+            return []
+
+        try:
+            data = json.loads(output)
+            return [f.get("path", "") for f in data.get("files", [])]
+        except json.JSONDecodeError:
+            return []
+
+    def _parse_datetime(self, dt_str: Optional[str]) -> Optional[datetime]:
+        """Parse ISO datetime string."""
+        if not dt_str:
+            return None
+        try:
+            # Handle ISO format with Z suffix
+            dt_str = dt_str.replace("Z", "+00:00")
+            return datetime.fromisoformat(dt_str)
+        except (ValueError, AttributeError):
+            return None
+
+    def _extract_pr_from_json(self, pr_data: dict) -> PullRequestEntity:
+        """Extract PR data from gh CLI JSON output.
+
+        Args:
+            pr_data: PR data from gh CLI
+
+        Returns:
+            PullRequestEntity with extracted data
+        """
+        # Determine state
+        state = pr_data.get("state", "open").lower()
+        if pr_data.get("mergedAt"):
+            state = "merged"
+
+        # Get reviewers
+        reviewers = set()
+        for req in pr_data.get("reviewRequests", []):
+            if isinstance(req, dict) and req.get("login"):
+                reviewers.add(req["login"])
+        for review in pr_data.get("reviews", []):
+            if isinstance(review, dict) and review.get("author", {}).get("login"):
+                reviewers.add(review["author"]["login"])
+
+        # Get commit SHAs
+        commit_shas = []
+        for commit in pr_data.get("commits", []):
+            if isinstance(commit, dict) and commit.get("oid"):
+                commit_shas.append(commit["oid"])
+
+        # Get files changed
+        files_changed = []
+        for f in pr_data.get("files", []):
+            if isinstance(f, dict) and f.get("path"):
+                files_changed.append(f["path"])
+
+        # Get labels
+        labels = []
+        for label in pr_data.get("labels", []):
+            if isinstance(label, dict) and label.get("name"):
+                labels.append(label["name"])
+
+        # Get author
+        author = "unknown"
+        author_data = pr_data.get("author")
+        if isinstance(author_data, dict):
+            author = author_data.get("login", "unknown")
+
+        return PullRequestEntity(
+            number=pr_data.get("number", 0),
+            title=pr_data.get("title", ""),
+            description=pr_data.get("body"),
+            state=state,
+            created_at=self._parse_datetime(pr_data.get("createdAt")),
+            author=author,
+            merged_at=self._parse_datetime(pr_data.get("mergedAt")),
+            reviewers=list(reviewers),
+            labels=labels,
+            additions=pr_data.get("additions", 0),
+            deletions=pr_data.get("deletions", 0),
+            files_changed=files_changed,
+            commit_shas=commit_shas,
+            base_branch=pr_data.get("baseRefName", "main"),
+            head_branch=pr_data.get("headRefName", ""),
+        )