solokit 0.1.1__py3-none-any.whl

solokit/learning/extractor.py

@@ -0,0 +1,352 @@
+"""Learning extraction module for various sources"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+from solokit.core.command_runner import CommandRunner
+from solokit.core.constants import GIT_STANDARD_TIMEOUT
+from solokit.core.error_handlers import log_errors
+from solokit.core.exceptions import FileOperationError
+from solokit.core.file_ops import load_json
+from solokit.core.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+
+class LearningExtractor:
+    """Extracts learnings from various sources (sessions, git commits, code comments)"""
+
+    def __init__(self, session_dir: Path, project_root: Path | None = None):
+        """
+        Initialize learning extractor
+
+        Args:
+            session_dir: Path to .session directory
+            project_root: Path to project root (for git operations)
+        """
+        self.session_dir = session_dir
+        self.project_root = project_root or Path.cwd()
+        self.runner = CommandRunner(
+            default_timeout=GIT_STANDARD_TIMEOUT, working_dir=self.project_root
+        )
+
+    def extract_from_sessions(self) -> list[dict[str, Any]]:
+        """
+        Extract learnings from session summary JSON files
+
+        Returns:
+            List of learning dictionaries
+        """
+        learnings: list[dict[str, Any]] = []
+        summaries_dir = self.session_dir / "summaries"
+
+        if not summaries_dir.exists():
+            return learnings
+
+        # Look for session summary files
+        for summary_file in summaries_dir.glob("session_*.json"):
+            try:
+                summary_data = load_json(summary_file)
+
+                # Extract learnings from various fields
+                session_id = summary_file.stem.replace("session_", "")
+
+                # Check for explicit learnings field
+                if "learnings" in summary_data:
+                    for learning_text in summary_data["learnings"]:
+                        learnings.append(
+                            {
+                                "content": learning_text,
+                                "learned_in": session_id,
+                                "timestamp": summary_data.get("timestamp", ""),
+                            }
+                        )
+
+                # Extract from challenges as potential gotchas
+                if "challenges_encountered" in summary_data:
+                    for challenge in summary_data["challenges_encountered"]:
+                        learnings.append(
+                            {
+                                "content": f"Challenge: {challenge}",
+                                "learned_in": session_id,
+                                "timestamp": summary_data.get("timestamp", ""),
+                                "suggested_type": "gotcha",
+                            }
+                        )
+
+            except (ValueError, KeyError, FileOperationError) as e:
+                # Skip invalid summary files
+                logger.warning(f"Failed to extract learnings from {summary_file}: {e}")
+                continue
+
+        logger.info(f"Extracted {len(learnings)} learnings from session summaries")
+        return learnings
+
+    @log_errors()
+    def extract_from_session_summary(
+        self, session_file: Path, validator: Any = None
+    ) -> list[dict[str, Any]]:
+        """
+        Extract learnings from a session summary markdown file
+
+        Args:
+            session_file: Path to session summary markdown file
+            validator: Optional validator instance with create_learning_entry and validate_learning methods
+
+        Returns:
+            List of learning dictionaries extracted from the file
+        """
+        if not session_file.exists():
+            return []
+
+        try:
+            with open(session_file) as f:
+                content = f.read()
+        except (OSError, Exception) as e:
+            logger.warning(f"Failed to read session summary {session_file}: {e}")
+            return []
+
+        learnings = []
+
+        # Extract session number from filename (e.g., session_005_summary.md)
+        session_match = re.search(r"session_(\d+)", session_file.name)
+        session_num = int(session_match.group(1)) if session_match else 0
+
+        # Look for "Challenges Encountered" or "Learnings Captured" sections
+        patterns = [
+            r"##\s*Challenges?\s*Encountered\s*\n(.*?)(?=\n##|\Z)",
+            r"##\s*Learnings?\s*Captured\s*\n(.*?)(?=\n##|\Z)",
+        ]
+
+        for pattern in patterns:
+            matches = re.findall(pattern, content, re.DOTALL | re.IGNORECASE)
+            for match in matches:
+                # Each bullet point is a potential learning
+                for line in match.split("\n"):
+                    line = line.strip()
+                    if line.startswith("-") or line.startswith("*"):
+                        learning_text = line.lstrip("-*").strip()
+                        # Basic validation
+                        if learning_text and self._is_valid_content(learning_text):
+                            if validator:
+                                # Use validator for standardized entry creation
+                                entry = validator.create_learning_entry(
+                                    content=learning_text,
+                                    source="session_summary",
+                                    session_id=f"session_{session_num:03d}",
+                                    context=f"Session summary file: {session_file.name}",
+                                )
+                                if validator.validate_learning(entry):
+                                    learnings.append(entry)
+                            else:
+                                # Simple entry without validation
+                                learnings.append(
+                                    {
+                                        "content": learning_text,
+                                        "learned_in": f"session_{session_num:03d}",
+                                        "source": "session_summary",
+                                        "context": f"Session summary file: {session_file.name}",
+                                    }
+                                )
+
+        logger.info(f"Extracted {len(learnings)} learnings from {session_file.name}")
+        return learnings
+
+    @log_errors()
+    def extract_from_git_commits(
+        self, since_session: int = 0, session_id: str | None = None, validator: Any = None
+    ) -> list[dict[str, Any]]:
+        """
+        Extract learnings from git commit messages with LEARNING: annotations
+
+        Args:
+            since_session: Extract only commits after this session number
+            session_id: Session ID to tag learnings with
+            validator: Optional validator instance
+
+        Returns:
+            List of learning dictionaries extracted from commit messages
+        """
+        try:
+            # Get recent commits
+            result = self.runner.run(["git", "log", "--format=%H|||%B", "-n", "100"])
+
+            if not result.success:
+                return []
+
+            learnings = []
+            # Updated regex to capture multi-line LEARNING statements
+            # Captures until: double newline (blank line) OR end of string
+            learning_pattern = r"LEARNING:\s*([\s\S]+?)(?=\n\n|\Z)"
+
+            # Parse commit messages
+            commits_raw = result.stdout.strip()
+            if not commits_raw:
+                return []
+
+            # Each commit starts with hash|||, split on newline followed by hash pattern
+            commit_blocks = re.split(r"\n(?=[a-f0-9]{40}\|\|\|)", commits_raw)
+
+            for commit_block in commit_blocks:
+                if "|||" not in commit_block:
+                    continue
+
+                commit_hash, message = commit_block.split("|||", 1)
+
+                # Find LEARNING: annotations
+                for match in re.finditer(learning_pattern, message, re.MULTILINE):
+                    learning_text = match.group(1).strip()
+                    # Basic validation
+                    if learning_text and self._is_valid_content(learning_text):
+                        if validator:
+                            # Use validator for standardized entry creation
+                            entry = validator.create_learning_entry(
+                                content=learning_text,
+                                source="git_commit",
+                                session_id=session_id,
+                                context=f"Commit {commit_hash[:8]}",
+                            )
+                            if validator.validate_learning(entry):
+                                learnings.append(entry)
+                        else:
+                            # Simple entry without validation
+                            learnings.append(
+                                {
+                                    "content": learning_text,
+                                    "learned_in": session_id or "unknown",
+                                    "source": "git_commit",
+                                    "context": f"Commit {commit_hash[:8]}",
+                                }
+                            )
+
+            logger.info(f"Extracted {len(learnings)} learnings from git commits")
+            return learnings
+
+        except Exception as e:
+            logger.warning(f"Failed to extract learnings from git commits: {e}")
+            return []
+
+    @log_errors()
+    def extract_from_code_comments(
+        self,
+        changed_files: list[Path] | None = None,
+        session_id: str | None = None,
+        validator: Any = None,
+    ) -> list[dict[str, Any]]:
+        """
+        Extract learnings from inline code comments (not documentation)
+
+        Args:
+            changed_files: List of file paths to scan (or None to auto-detect from git)
+            session_id: Session ID to tag learnings with
+            validator: Optional validator instance
+
+        Returns:
+            List of learning dictionaries extracted from code comments
+        """
+        if changed_files is None:
+            # Get recently changed files from git
+            try:
+                result = self.runner.run(["git", "diff", "--name-only", "HEAD~5", "HEAD"])
+
+                if result.success:
+                    changed_files = [
+                        self.project_root / f.strip()
+                        for f in result.stdout.split("\n")
+                        if f.strip()
+                    ]
+                else:
+                    changed_files = []
+            except Exception as e:
+                logger.warning(f"Failed to get changed files from git: {e}")
+                changed_files = []
+
+        learnings = []
+        # Pattern must match actual comment lines (starting with #), not string literals
+        learning_pattern = r"^\s*#\s*LEARNING:\s*(.+?)$"
+
+        # Only scan actual code files, not documentation
+        code_extensions = {".py", ".js", ".ts", ".jsx", ".tsx", ".go", ".rs"}
+        doc_extensions = {".md", ".txt", ".rst"}
+        excluded_dirs = {"examples", "templates", "tests", "test", "__tests__", "spec"}
+
+        for file_path in changed_files:
+            if not file_path.exists() or not file_path.is_file():
+                continue
+
+            # Skip documentation files
+            if file_path.suffix in doc_extensions:
+                continue
+
+            # Skip example/template/test directories
+            if any(excluded_dir in file_path.parts for excluded_dir in excluded_dirs):
+                continue
+
+            # Only process code files
+            if file_path.suffix not in code_extensions:
+                continue
+
+            # Skip binary files and large files
+            if file_path.stat().st_size > 1_000_000:
+                continue
+
+            try:
+                with open(file_path, encoding="utf-8", errors="ignore") as f:
+                    for line_num, line in enumerate(f, 1):
+                        match = re.search(learning_pattern, line)
+                        if match:
+                            learning_text = match.group(1).strip()
+                            # Basic validation
+                            if learning_text and self._is_valid_content(learning_text):
+                                if validator:
+                                    # Use validator for standardized entry creation
+                                    entry = validator.create_learning_entry(
+                                        content=learning_text,
+                                        source="inline_comment",
+                                        session_id=session_id,
+                                        context=f"{file_path.name}:{line_num}",
+                                    )
+                                    if validator.validate_learning(entry):
+                                        learnings.append(entry)
+                                else:
+                                    # Simple entry without validation
+                                    learnings.append(
+                                        {
+                                            "content": learning_text,
+                                            "learned_in": session_id or "unknown",
+                                            "source": "inline_comment",
+                                            "context": f"{file_path.name}:{line_num}",
+                                        }
+                                    )
+            except (OSError, UnicodeDecodeError) as e:
+                logger.warning(f"Failed to read file {file_path}: {e}")
+                continue
+
+        logger.info(f"Extracted {len(learnings)} learnings from code comments")
+        return learnings
+
+    def _is_valid_content(self, content: str) -> bool:
+        """
+        Basic validation for learning content
+
+        Args:
+            content: Content to validate
+
+        Returns:
+            True if content appears valid
+        """
+        if not content or not isinstance(content, str):
+            return False
+
+        # Skip placeholders and examples
+        if "<" in content or ">" in content:
+            return False
+
+        # Must have substance (more than just a few words)
+        if len(content.split()) < 5:
+            return False
+
+        return True

solokit/learning/reporter.py

@@ -0,0 +1,351 @@
+"""Learning reporting and statistics module"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from solokit.core.logging_config import get_logger
+from solokit.core.output import get_output
+
+logger = get_logger(__name__)
+output = get_output()
+
+
+class LearningReporter:
+    """Handles learning reports, statistics, searches, and display functionality"""
+
+    def __init__(self, repository: Any):
+        """
+        Initialize reporter
+
+        Args:
+            repository: LearningRepository instance for data access
+        """
+        self.repository = repository
+
+    def generate_report(self) -> None:
+        """Generate learning summary report"""
+        output.section("Learning Summary Report")
+
+        learnings = self.repository.load_learnings()
+
+        # Create table
+        output.info("Learnings by Category:")
+        output.info("-" * 40)
+
+        categories = learnings.get("categories", {})
+        total = 0
+
+        for category_name, category_learnings in categories.items():
+            count = len(category_learnings)
+            total += count
+            formatted_name = category_name.replace("_", " ").title()
+            output.info(f"{formatted_name:<30} {count:>5}")
+
+        # Add archived
+        archived_count = len(learnings.get("archived", []))
+        if archived_count > 0:
+            output.info(f"{'Archived':<30} {archived_count:>5}")
+
+        # Add total
+        output.info("-" * 40)
+        output.info(f"{'Total':<30} {total:>5}")
+        output.info("")
+
+        # Show last curated
+        last_curated = learnings.get("last_curated")
+        if last_curated:
+            output.info(f"Last curated: {last_curated}\n")
+        else:
+            output.info("Never curated\n")
+
+    def search_learnings(self, query: str) -> None:
+        """
+        Search learnings by keyword
+
+        Args:
+            query: Search query string
+        """
+        learnings = self.repository.load_learnings()
+        categories = learnings.get("categories", {})
+
+        query_lower = query.lower()
+        matches = []
+
+        # Search through all learnings
+        for category_name, category_learnings in categories.items():
+            for learning in category_learnings:
+                # Search in content
+                content = learning.get("content", "").lower()
+                tags = learning.get("tags", [])
+                context = learning.get("context", "").lower()
+
+                if (
+                    query_lower in content
+                    or query_lower in context
+                    or any(query_lower in tag.lower() for tag in tags)
+                ):
+                    matches.append({**learning, "category": category_name})
+
+        # Display results
+        if not matches:
+            output.info(f"\nNo learnings found matching '{query}'\n")
+            return
+
+        output.info(f"\n=== Search Results for '{query}' ===\n")
+        output.info(f"Found {len(matches)} matching learning(s):\n")
+
+        for i, learning in enumerate(matches, 1):
+            output.info(f"{i}. [{learning['category'].replace('_', ' ').title()}]")
+            output.info(f"   {learning['content']}")
+
+            if "tags" in learning:
+                output.info(f"   Tags: {', '.join(learning['tags'])}")
+
+            output.info(f"   Session: {learning.get('learned_in', 'unknown')}")
+            output.info(f"   ID: {learning.get('id', 'N/A')}")
+            output.info("")
+
+    def show_learnings(
+        self,
+        category: str | None = None,
+        tag: str | None = None,
+        session: int | None = None,
+        date_from: str | None = None,
+        date_to: str | None = None,
+        include_archived: bool = False,
+    ) -> None:
+        """
+        Show learnings with optional filters
+
+        Args:
+            category: Filter by category name
+            tag: Filter by tag
+            session: Filter by session number
+            date_from: Filter by start date
+            date_to: Filter by end date
+            include_archived: Include archived learnings
+        """
+        learnings = self.repository.load_learnings()
+        categories = learnings.get("categories", {})
+
+        # Apply filters
+        filtered = []
+        for category_name, category_learnings in categories.items():
+            # Category filter
+            if category and category_name != category:
+                continue
+
+            for learning in category_learnings:
+                # Tag filter
+                if tag and tag not in learning.get("tags", []):
+                    continue
+
+                # Session filter
+                if session:
+                    learned_in = learning.get("learned_in", "")
+                    session_num = self._extract_session_number(learned_in)
+                    if session_num != session:
+                        continue
+
+                # Date range filter
+                if date_from or date_to:
+                    learning_date = learning.get("timestamp", "")
+                    if date_from and learning_date < date_from:
+                        continue
+                    if date_to and learning_date > date_to:
+                        continue
+
+                filtered.append({**learning, "category": category_name})
+
+        # Display results
+        if not filtered:
+            output.info("\nNo learnings found matching the filters\n")
+            return
+
+        if category:
+            # Show specific category
+            output.info(f"\n{category.replace('_', ' ').title()}\n")
+            output.info("=" * 50)
+
+            for i, learning in enumerate(filtered, 1):
+                output.info(f"\n{i}. {learning.get('content', 'N/A')}")
+                if "tags" in learning:
+                    output.info(f"   Tags: {', '.join(learning['tags'])}")
+                if "learned_in" in learning:
+                    output.info(f"   Learned in: {learning['learned_in']}")
+                if "timestamp" in learning:
+                    output.info(f"   Date: {learning['timestamp']}")
+                output.info(f"   ID: {learning.get('id', 'N/A')}")
+        else:
+            # Show all categories
+            grouped: dict[str, list[Any]] = {}
+            for learning in filtered:
+                cat = learning["category"]
+                if cat not in grouped:
+                    grouped[cat] = []
+                grouped[cat].append(learning)
+
+            for category_name, category_learnings in grouped.items():
+                output.info(f"\n{category_name.replace('_', ' ').title()}")
+                output.info(f"Count: {len(category_learnings)}\n")
+
+                # Show first 3
+                for learning in category_learnings[:3]:
+                    output.info(f"  • {learning.get('content', 'N/A')}")
+                    if "tags" in learning:
+                        output.info(f"    Tags: {', '.join(learning['tags'])}")
+
+                if len(category_learnings) > 3:
+                    output.info(f"  ... and {len(category_learnings) - 3} more")
+
+                output.info("")
+
+    def generate_statistics(self) -> dict[str, Any]:
+        """
+        Generate learning statistics
+
+        Returns:
+            Dictionary with statistics (total, by_category, by_tag, top_tags, by_session)
+        """
+        learnings = self.repository.load_learnings()
+        categories = learnings.get("categories", {})
+
+        stats: dict[str, Any] = {
+            "total": 0,
+            "by_category": {},
+            "by_tag": {},
+            "top_tags": [],
+            "by_session": {},
+        }
+
+        # Count by category
+        for cat, items in categories.items():
+            count = len(items)
+            stats["by_category"][cat] = count
+            stats["total"] += count
+
+        # Count by tag and session
+        tag_counts: dict[str, int] = {}
+        session_counts: dict[int, int] = {}
+
+        for items in categories.values():
+            for learning in items:
+                # Tag counts
+                for tag in learning.get("tags", []):
+                    tag_counts[tag] = tag_counts.get(tag, 0) + 1
+
+                # Session counts
+                learned_in = learning.get("learned_in", "unknown")
+                session_num = self._extract_session_number(learned_in)
+                if session_num > 0:
+                    session_counts[session_num] = session_counts.get(session_num, 0) + 1
+
+        # Top tags
+        stats["top_tags"] = sorted(tag_counts.items(), key=lambda x: x[1], reverse=True)[:10]
+        stats["by_tag"] = tag_counts
+        stats["by_session"] = session_counts
+
+        return stats
+
+    def show_statistics(self) -> None:
+        """Display learning statistics"""
+        stats = self.generate_statistics()
+
+        output.section("Learning Statistics")
+
+        # Total
+        output.info(f"Total learnings: {stats['total']}\n")
+
+        # By category
+        output.info("By Category:")
+        output.info("-" * 40)
+        for cat, count in stats["by_category"].items():
+            cat_name = cat.replace("_", " ").title()
+            output.info(f"  {cat_name:<30} {count:>5}")
+
+        # Top tags
+        if stats["top_tags"]:
+            output.info("\nTop Tags:")
+            output.info("-" * 40)
+            for tag, count in stats["top_tags"]:
+                output.info(f"  {tag:<30} {count:>5}")
+
+        # Sessions with most learnings
+        if stats["by_session"]:
+            top_sessions = sorted(stats["by_session"].items(), key=lambda x: x[1], reverse=True)[:5]
+            output.info("\nSessions with Most Learnings:")
+            output.info("-" * 40)
+            for session_num, count in top_sessions:
+                output.info(f"  Session {session_num:<22} {count:>5}")
+
+        output.info("")
+
+    def show_timeline(self, sessions: int = 10) -> None:
+        """
+        Show learning timeline for recent sessions
+
+        Args:
+            sessions: Number of recent sessions to display
+        """
+        learnings = self.repository.load_learnings()
+        categories = learnings.get("categories", {})
+
+        # Group by session
+        by_session: dict[int, list[Any]] = {}
+        for items in categories.values():
+            for learning in items:
+                learned_in = learning.get("learned_in", "unknown")
+                session = self._extract_session_number(learned_in)
+                if session > 0:
+                    if session not in by_session:
+                        by_session[session] = []
+                    by_session[session].append(learning)
+
+        if not by_session:
+            output.info("\nNo session timeline available\n")
+            return
+
+        # Display recent sessions
+        recent = sorted(by_session.keys(), reverse=True)[:sessions]
+
+        output.info(f"\n=== Learning Timeline (Last {min(len(recent), sessions)} Sessions) ===\n")
+
+        for session in recent:
+            session_learnings = by_session[session]
+            count = len(session_learnings)
+
+            output.info(f"Session {session:03d}: {count} learning(s)")
+
+            # Show first 3 learnings
+            for learning in session_learnings[:3]:
+                content = learning.get("content", "")
+                # Truncate long learnings
+                if len(content) > 60:
+                    content = content[:57] + "..."
+                output.info(f"  - {content}")
+
+            if len(session_learnings) > 3:
+                output.info(f"  ... and {len(session_learnings) - 3} more")
+
+            output.info("")
+
+    def _extract_session_number(self, session_id: str) -> int:
+        """
+        Extract numeric session number from session ID
+
+        Args:
+            session_id: Session ID string
+
+        Returns:
+            Extracted session number or 0
+        """
+        try:
+            match = re.search(r"\d+", session_id)
+            if match:
+                return int(match.group())
+        except (ValueError, AttributeError) as e:
+            logger.debug(f"Failed to extract session number from '{session_id}': {e}")
+            return 0
+        return 0