brainlayer 1.0.0__py3-none-any.whl

brainlayer/pipeline/extract_whatsapp.py

@@ -0,0 +1,227 @@
+"""Extract WhatsApp messages for communication pattern analysis.
+
+Extracts messages from WhatsApp's SQLite database to analyze:
+- User's writing style and patterns
+- Response patterns
+- Communication preferences
+"""
+
+import sqlite3
+from datetime import datetime
+from pathlib import Path
+from typing import Iterator, Optional
+
+
+def get_whatsapp_db_path() -> Path:
+    """Get path to WhatsApp's ChatStorage database."""
+    return Path.home() / "Library/Group Containers/group.net.whatsapp.WhatsApp.shared/ChatStorage.sqlite"
+
+
+def get_contacts_db_path() -> Path:
+    """Get path to WhatsApp's Contacts database."""
+    return Path.home() / "Library/Group Containers/group.net.whatsapp.WhatsApp.shared/ContactsV2.sqlite"
+
+
+def extract_whatsapp_messages(
+    db_path: Optional[Path] = None,
+    limit: Optional[int] = None,
+    only_from_me: bool = False,
+    exclude_groups: bool = True,
+    min_char_count: int = 10,
+) -> Iterator[dict]:
+    """
+    Extract WhatsApp messages from SQLite database.
+
+    Args:
+        db_path: Path to ChatStorage.sqlite (auto-detected if None)
+        limit: Maximum number of messages to extract
+        only_from_me: Only extract messages sent by user
+        exclude_groups: Exclude group chat messages
+        min_char_count: Minimum message length to include (filters emoji-only etc.)
+
+    Yields:
+        Message dictionaries with text, timestamp, sender info
+    """
+    if db_path is None:
+        db_path = get_whatsapp_db_path()
+
+    if not db_path.exists():
+        raise FileNotFoundError(f"WhatsApp database not found at: {db_path}")
+
+    # Open database in read-only mode
+    conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
+    conn.row_factory = sqlite3.Row
+
+    try:
+        cursor = conn.cursor()
+
+        # Build query
+        query = """
+            SELECT
+                Z_PK as id,
+                ZTEXT as text,
+                ZISFROMME as is_from_me,
+                ZMESSAGEDATE as message_date,
+                ZSENTDATE as sent_date,
+                ZFROMJID as from_jid,
+                ZTOJID as to_jid,
+                ZPUSHNAME as push_name,
+                ZMESSAGETYPE as message_type,
+                ZMESSAGESTATUS as message_status,
+                ZSTARRED as starred,
+                ZSTANZAID as stanza_id
+            FROM ZWAMESSAGE
+            WHERE ZTEXT IS NOT NULL
+                AND ZTEXT != ''
+                AND LENGTH(ZTEXT) >= ?
+        """
+        params = [min_char_count]
+
+        # Add filters
+        if only_from_me:
+            query += " AND ZISFROMME = 1"
+
+        if exclude_groups:
+            # Group JIDs typically contain '@g.us'
+            query += " AND (ZTOJID NOT LIKE '%@g.us' OR ZTOJID IS NULL)"
+
+        # Order by date descending (most recent first)
+        query += " ORDER BY ZMESSAGEDATE DESC"
+
+        if limit:
+            query += " LIMIT ?"
+            params.append(limit)
+
+        cursor.execute(query, params)
+
+        for row in cursor:
+            # Convert Core Data timestamp to Unix timestamp
+            # Core Data uses seconds since 2001-01-01
+            core_data_epoch = 978307200  # Unix timestamp for 2001-01-01
+            message_timestamp = row["message_date"] + core_data_epoch if row["message_date"] else None
+
+            yield {
+                "id": row["id"],
+                "text": row["text"],
+                "is_from_me": bool(row["is_from_me"]),
+                "timestamp": message_timestamp,
+                "datetime": datetime.fromtimestamp(message_timestamp) if message_timestamp else None,
+                "from_jid": row["from_jid"],
+                "to_jid": row["to_jid"],
+                "contact_name": row["push_name"],
+                "message_type": row["message_type"],
+                "starred": bool(row["starred"]),
+                "stanza_id": row["stanza_id"],
+            }
+
+    finally:
+        conn.close()
+
+
+def get_contact_info(jid: str, contacts_db_path: Optional[Path] = None) -> Optional[dict]:
+    """
+    Get contact information for a JID.
+
+    Args:
+        jid: WhatsApp JID (e.g., "1234567890@s.whatsapp.net")
+        contacts_db_path: Path to ContactsV2.sqlite
+
+    Returns:
+        Contact info dict or None if not found
+    """
+    if contacts_db_path is None:
+        contacts_db_path = get_contacts_db_path()
+
+    if not contacts_db_path.exists():
+        return None
+
+    conn = sqlite3.connect(f"file:{contacts_db_path}?mode=ro", uri=True)
+    conn.row_factory = sqlite3.Row
+
+    try:
+        cursor = conn.cursor()
+        cursor.execute(
+            """
+            SELECT
+                ZCONTACTNAME as name,
+                ZPHONENUMBER as phone,
+                ZSTATUSTEXT as status
+            FROM ZWACONTACT
+            WHERE ZCONTACTJID = ?
+        """,
+            (jid,),
+        )
+
+        row = cursor.fetchone()
+        if row:
+            return {
+                "name": row["name"],
+                "phone": row["phone"],
+                "status": row["status"],
+            }
+        return None
+
+    finally:
+        conn.close()
+
+
+def analyze_writing_style(messages: list[dict]) -> dict:
+    """
+    Analyze user's writing style from messages.
+
+    Returns patterns like:
+    - Average message length
+    - Common phrases
+    - Punctuation usage
+    - Emoji usage
+    - Response time patterns
+    """
+    user_messages = [m for m in messages if m["is_from_me"]]
+
+    if not user_messages:
+        return {}
+
+    total_length = sum(len(m["text"]) for m in user_messages)
+    avg_length = total_length / len(user_messages)
+
+    # Count emoji usage
+    emoji_count = sum(
+        1
+        for m in user_messages
+        for char in m["text"]
+        if ord(char) > 0x1F300  # Basic emoji range
+    )
+
+    # Analyze punctuation
+    exclamation_count = sum(m["text"].count("!") for m in user_messages)
+    question_count = sum(m["text"].count("?") for m in user_messages)
+
+    return {
+        "total_messages": len(user_messages),
+        "avg_message_length": avg_length,
+        "emoji_usage_rate": emoji_count / len(user_messages),
+        "exclamation_rate": exclamation_count / len(user_messages),
+        "question_rate": question_count / len(user_messages),
+        "sample_messages": [m["text"] for m in user_messages[:10]],
+    }
+
+
+def format_whatsapp_for_pipeline(message: dict) -> dict:
+    """
+    Format WhatsApp message to match brainlayer pipeline format.
+
+    Converts to a format similar to Claude Code conversations.
+    """
+    return {
+        "type": "whatsapp_message",
+        "id": f"whatsapp_{message['id']}",
+        "timestamp": message["timestamp"],
+        "role": "user" if message["is_from_me"] else "contact",
+        "content": message["text"],
+        "metadata": {
+            "source": "whatsapp",
+            "contact": message["contact_name"],
+            "jid": message["from_jid"] if not message["is_from_me"] else message["to_jid"],
+            "starred": message["starred"],
+        },
+    }

brainlayer/pipeline/git_overlay.py

@@ -0,0 +1,301 @@
+"""Git Overlay Pipeline — Cross-reference sessions with git history.
+
+Phase 8b: For each indexed session, extract git metadata (branch, commits,
+PR number, files changed) and build file-interaction timelines.
+
+Usage:
+    from brainlayer.pipeline.git_overlay import run_git_overlay
+    run_git_overlay(vector_store, project="-Users-username-Projects-myapp")
+"""
+
+import json
+import logging
+import re
+import subprocess
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+logger = logging.getLogger(__name__)
+
+
+def slug_to_path(slug: str) -> Optional[Path]:
+    """Convert a project slug to a filesystem path.
+
+    Example: '-Users-username-Projects-myapp' -> '/Users/username/Projects/myapp'
+    """
+    if not slug:
+        return None
+    # Replace leading '-' with '/', then '-' with '/'
+    path_str = "/" + slug.lstrip("-").replace("-", "/")
+    path = Path(path_str)
+    if path.exists() and path.is_dir():
+        return path
+    return None
+
+
+def slug_to_display(slug: str) -> str:
+    """Convert project slug to display name.
+
+    Example: '-Users-username-Projects-myapp' -> 'Projects/myapp'
+    """
+    return re.sub(r"^-Users-[^-]+-", "", slug).replace("-", "/")
+
+
+def _git_cmd(cmd: List[str], cwd: Path, timeout: int = 10) -> Optional[str]:
+    """Run a git command, return stdout or None on failure."""
+    try:
+        result = subprocess.run(
+            ["git"] + cmd,
+            cwd=str(cwd),
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+        if result.returncode == 0:
+            return result.stdout.strip()
+        return None
+    except (subprocess.TimeoutExpired, FileNotFoundError):
+        return None
+
+
+def is_git_repo(path: Path) -> bool:
+    """Check if path is inside a git repository."""
+    return _git_cmd(["rev-parse", "--git-dir"], path) is not None
+
+
+def get_session_timestamps(jsonl_path: Path) -> Tuple[Optional[str], Optional[str]]:
+    """Extract first and last timestamps from a session JSONL file."""
+    first_ts = None
+    last_ts = None
+    try:
+        with open(jsonl_path) as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    obj = json.loads(line)
+                    ts = obj.get("timestamp")
+                    if ts:
+                        if first_ts is None:
+                            first_ts = ts
+                        last_ts = ts
+                except json.JSONDecodeError:
+                    continue
+    except (OSError, IOError):
+        pass
+    return first_ts, last_ts
+
+
+def extract_file_actions(jsonl_path: Path, session_id: str) -> List[Dict[str, Any]]:
+    """Extract file interaction records from a session JSONL.
+
+    Looks for tool_use blocks in assistant messages with file paths.
+    """
+    interactions: List[Dict[str, Any]] = []
+    tool_to_action = {
+        "Read": "read",
+        "Edit": "edit",
+        "Write": "write",
+        "Glob": "search",
+        "Grep": "search",
+    }
+
+    try:
+        with open(jsonl_path) as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    obj = json.loads(line)
+                    if obj.get("type") != "assistant":
+                        continue
+                    msg = obj.get("message", {})
+                    content = msg.get("content", [])
+                    timestamp = obj.get("timestamp")
+                    if not isinstance(content, list):
+                        continue
+                    for block in content:
+                        if not isinstance(block, dict):
+                            continue
+                        if block.get("type") != "tool_use":
+                            continue
+                        tool_name = block.get("name", "")
+                        action = tool_to_action.get(tool_name)
+                        if not action:
+                            continue
+                        inp = block.get("input", {})
+                        file_path = inp.get("file_path") or inp.get("path")
+                        if file_path:
+                            interactions.append(
+                                {
+                                    "file_path": file_path,
+                                    "timestamp": timestamp,
+                                    "session_id": session_id,
+                                    "action": action,
+                                }
+                            )
+                except json.JSONDecodeError:
+                    continue
+    except (OSError, IOError):
+        pass
+    return interactions
+
+
+def get_git_context(
+    repo_path: Path,
+    start_ts: str,
+    end_ts: str,
+) -> Dict[str, Any]:
+    """Get git context for a time range in a repo.
+
+    Returns branch, commits, PR number, and files changed.
+    """
+    context: Dict[str, Any] = {
+        "branch": None,
+        "pr_number": None,
+        "commit_shas": [],
+        "files_changed": [],
+    }
+
+    # Current branch (at time of overlay, not session time)
+    branch = _git_cmd(["rev-parse", "--abbrev-ref", "HEAD"], repo_path)
+    context["branch"] = branch
+
+    # Commits in the time range
+    log_output = _git_cmd(
+        ["log", f"--after={start_ts}", f"--before={end_ts}", "--format=%H %s", "--all"],
+        repo_path,
+    )
+    if log_output:
+        for line in log_output.strip().split("\n"):
+            if line.strip():
+                parts = line.split(" ", 1)
+                context["commit_shas"].append(parts[0])
+
+    # Files changed in those commits
+    if context["commit_shas"]:
+        first_sha = context["commit_shas"][-1]  # oldest
+        last_sha = context["commit_shas"][0]  # newest
+        diff_output = _git_cmd(
+            ["diff", "--name-only", f"{first_sha}~1", last_sha],
+            repo_path,
+        )
+        if diff_output:
+            context["files_changed"] = [f for f in diff_output.strip().split("\n") if f.strip()]
+
+    # Try to extract PR number from branch name
+    if branch and branch not in ("master", "main", "HEAD"):
+        # Try gh CLI for PR number
+        try:
+            result = subprocess.run(
+                ["gh", "pr", "list", "--head", branch, "--json", "number", "--limit", "1"],
+                cwd=str(repo_path),
+                capture_output=True,
+                text=True,
+                timeout=10,
+            )
+            if result.returncode == 0:
+                prs = json.loads(result.stdout)
+                if prs:
+                    context["pr_number"] = prs[0]["number"]
+        except (subprocess.TimeoutExpired, FileNotFoundError, json.JSONDecodeError):
+            pass
+
+    return context
+
+
+def run_git_overlay(
+    vector_store: Any,
+    project: Optional[str] = None,
+    force: bool = False,
+    max_sessions: int = 0,
+) -> Dict[str, int]:
+    """Run git overlay on indexed sessions.
+
+    Args:
+        vector_store: VectorStore instance
+        project: Filter to specific project slug
+        force: Re-process sessions that already have context
+        max_sessions: Limit number of sessions (0 = all)
+
+    Returns:
+        Dict with counts: sessions_processed, file_interactions_added
+    """
+    projects_dir = Path.home() / ".claude" / "projects"
+    if not projects_dir.exists():
+        logger.warning("No projects directory found at %s", projects_dir)
+        return {"sessions_processed": 0, "file_interactions_added": 0}
+
+    stats = {"sessions_processed": 0, "file_interactions_added": 0}
+    processed = 0
+
+    for proj_dir in sorted(projects_dir.iterdir()):
+        if not proj_dir.is_dir():
+            continue
+        if project and proj_dir.name != project:
+            continue
+
+        # Resolve project path
+        repo_path = slug_to_path(proj_dir.name)
+        display_name = slug_to_display(proj_dir.name)
+        has_git = repo_path and is_git_repo(repo_path)
+
+        for jsonl_file in sorted(proj_dir.glob("*.jsonl")):
+            if max_sessions and processed >= max_sessions:
+                break
+
+            session_id = jsonl_file.stem
+
+            # Skip if already processed (unless force)
+            if not force:
+                existing = vector_store.get_session_context(session_id)
+                if existing:
+                    continue
+
+            # Get timestamps
+            start_ts, end_ts = get_session_timestamps(jsonl_file)
+            if not start_ts:
+                continue
+
+            # Extract file interactions from JSONL
+            file_actions = extract_file_actions(jsonl_file, session_id)
+            for fa in file_actions:
+                fa["project"] = display_name
+
+            # Get git context if repo exists
+            git_ctx = {}
+            if has_git and repo_path:
+                git_ctx = get_git_context(repo_path, start_ts, end_ts)
+
+            # Store session context
+            vector_store.store_session_context(
+                session_id=session_id,
+                project=display_name,
+                branch=git_ctx.get("branch"),
+                pr_number=git_ctx.get("pr_number"),
+                commit_shas=git_ctx.get("commit_shas", []),
+                files_changed=git_ctx.get("files_changed", []),
+                started_at=start_ts,
+                ended_at=end_ts,
+            )
+
+            # Store file interactions
+            count = vector_store.store_file_interactions(file_actions)
+            stats["file_interactions_added"] += count
+            stats["sessions_processed"] += 1
+            processed += 1
+
+            logger.info(
+                "Session %s: %s files, %d interactions, branch=%s",
+                session_id[:8],
+                len(git_ctx.get("files_changed", [])),
+                count,
+                git_ctx.get("branch", "n/a"),
+            )
+
+        if max_sessions and processed >= max_sessions:
+            break
+
+    return stats