brainlayer 1.0.0__py3-none-any.whl

brainlayer/pipeline/extract.py

@@ -0,0 +1,66 @@
+"""Stage 1: Extract system prompts and detect conversation continuations."""
+
+import hashlib
+from pathlib import Path
+from typing import Iterator
+
+import orjson
+
+
+def hash_content(content: str) -> str:
+    """SHA-256 hash for content-addressable storage."""
+    return hashlib.sha256(content.encode()).hexdigest()
+
+
+def parse_jsonl(file_path: Path) -> Iterator[dict]:
+    """Parse a JSONL file, yielding each line as a dict."""
+    with open(file_path, "rb") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                yield orjson.loads(line)
+            except orjson.JSONDecodeError:
+                continue  # Skip malformed lines
+
+
+def extract_system_prompts(conversations_dir: Path) -> dict[str, str]:
+    """
+    Extract and deduplicate system prompts from all conversations.
+
+    Returns: {hash: prompt_content} mapping for content-addressable storage.
+    """
+    prompts: dict[str, str] = {}
+
+    for jsonl_file in conversations_dir.rglob("*.jsonl"):
+        for entry in parse_jsonl(jsonl_file):
+            # First user message often contains system prompt
+            if entry.get("type") == "user":
+                message = entry.get("message", {})
+                content = message.get("content", "")
+
+                # Heuristic: system prompts are typically >2000 chars
+                # and contain CLAUDE.md or system instructions
+                if len(content) > 2000 and ("CLAUDE.md" in content or "system" in content.lower()):
+                    content_hash = hash_content(content)
+                    if content_hash not in prompts:
+                        prompts[content_hash] = content
+
+    return prompts
+
+
+def detect_continuations(conversations_dir: Path) -> list[list[Path]]:
+    """
+    Detect conversation continuations across multiple JSONL files.
+
+    Uses:
+    - Session ID matching
+    - Temporal proximity (within 30 min)
+    - Same project directory
+
+    Returns: List of continuation chains (each chain is a list of file paths).
+    """
+    # TODO: Implement continuation detection
+    # For now, treat each file as independent
+    return [[f] for f in conversations_dir.rglob("*.jsonl")]

brainlayer/pipeline/extract_claude_desktop.py

@@ -0,0 +1,149 @@
+"""Extract Claude chat transcripts from desktop/web app IndexedDB.
+
+Claude desktop app stores conversations in LevelDB (IndexedDB backend).
+This module extracts chat data for analysis.
+"""
+
+import json
+from pathlib import Path
+from typing import Iterator
+
+try:
+    import plyvel
+
+    HAS_PLYVEL = True
+except ImportError:
+    HAS_PLYVEL = False
+
+
+def get_claude_indexeddb_path() -> Path:
+    """Get path to Claude desktop app's IndexedDB."""
+    return Path.home() / "Library/Application Support/Claude/IndexedDB/https_claude.ai_0.indexeddb.leveldb"
+
+
+def extract_with_plyvel(db_path: Path) -> Iterator[dict]:
+    """Extract data using plyvel (requires: pip install plyvel)."""
+    if not HAS_PLYVEL:
+        raise ImportError("plyvel not installed. Run: pip install plyvel")
+
+    db = plyvel.DB(str(db_path), create_if_missing=False)
+
+    try:
+        for key, value in db:
+            # IndexedDB stores data with metadata prefixes
+            # We need to parse the binary format
+            try:
+                # Try to decode as JSON (some values are JSON strings)
+                decoded = value.decode("utf-8", errors="ignore")
+                if decoded.startswith("{") or decoded.startswith("["):
+                    data = json.loads(decoded)
+                    yield data
+            except (json.JSONDecodeError, UnicodeDecodeError):
+                continue
+    finally:
+        db.close()
+
+
+def extract_with_chrome_devtools() -> Iterator[dict]:
+    """
+    Extract using Chrome DevTools Protocol (alternative method).
+
+    This requires the Claude desktop app to be running and accessible.
+    More reliable than parsing LevelDB directly.
+    """
+    # TODO: Implement CDP extraction
+    # This would connect to the running app and extract via JavaScript
+    raise NotImplementedError("Chrome DevTools extraction not yet implemented")
+
+
+def extract_conversations_from_export(export_path: Path) -> Iterator[dict]:
+    """
+    Extract from Claude chat export (if available).
+
+    Claude.ai may offer export functionality - this handles that format.
+    """
+    if not export_path.exists():
+        raise FileNotFoundError(f"Export file not found: {export_path}")
+
+    with open(export_path, "r") as f:
+        data = json.load(f)
+
+    # Format depends on Claude's export structure
+    # Assuming it's a list of conversations
+    if isinstance(data, list):
+        yield from data
+    elif isinstance(data, dict) and "conversations" in data:
+        yield from data["conversations"]
+
+
+def extract_claude_chats(method: str = "manual") -> Iterator[dict]:
+    """
+    Extract Claude chat transcripts.
+
+    Args:
+        method: Extraction method
+            - "manual": User manually exports from Claude.ai
+            - "plyvel": Direct LevelDB access (requires plyvel)
+            - "cdp": Chrome DevTools Protocol (requires running app)
+
+    Yields:
+        Chat conversation dictionaries
+    """
+    if method == "manual":
+        # Guide user to export manually
+        print("\n" + "=" * 70)
+        print("MANUAL EXPORT INSTRUCTIONS")
+        print("=" * 70)
+        print("\n1. Open Claude.ai in your browser")
+        print("2. Go to Settings > Data & Privacy")
+        print("3. Click 'Export my data'")
+        print("4. Download the export file")
+        print("5. Save it to: ~/.local/share/brainlayer/claude-export.json")
+        print("\nAlternatively, copy conversations manually from the web interface.")
+        print("=" * 70 + "\n")
+
+        export_path = Path.home() / ".local/share/brainlayer/claude-export.json"
+        if export_path.exists():
+            yield from extract_conversations_from_export(export_path)
+        else:
+            print(f"Waiting for export at: {export_path}")
+            return
+
+    elif method == "plyvel":
+        db_path = get_claude_indexeddb_path()
+        if not db_path.exists():
+            raise FileNotFoundError(f"Claude IndexedDB not found at: {db_path}")
+        yield from extract_with_plyvel(db_path)
+
+    elif method == "cdp":
+        yield from extract_with_chrome_devtools()
+
+    else:
+        raise ValueError(f"Unknown extraction method: {method}")
+
+
+def format_claude_chat_for_pipeline(conversation: dict) -> dict:
+    """
+    Format Claude chat data to match brainlayer pipeline format.
+
+    Converts Claude's chat format to the JSONL format used by Claude Code.
+    """
+    # This depends on the actual format from Claude.ai
+    # Placeholder structure:
+    return {
+        "type": "conversation",
+        "id": conversation.get("id"),
+        "created_at": conversation.get("created_at"),
+        "messages": [
+            {
+                "role": msg.get("role", "user"),
+                "content": msg.get("content", ""),
+                "timestamp": msg.get("timestamp"),
+            }
+            for msg in conversation.get("messages", [])
+        ],
+        "metadata": {
+            "source": "claude_desktop",
+            "model": conversation.get("model"),
+        },
+    }

brainlayer/pipeline/extract_corrections.py

@@ -0,0 +1,231 @@
+"""Extract correction patterns from Claude/Gemini conversations.
+
+Identifies where AI assistants helped draft text and user made corrections,
+to learn what the user consistently changes.
+"""
+
+import json
+import re
+from pathlib import Path
+from typing import Iterator
+
+
+def extract_claude_export_conversations(export_path: Path) -> Iterator[dict]:
+    """
+    Extract conversations from Claude.ai export.
+
+    Args:
+        export_path: Path to extracted conversations.json
+
+    Yields:
+        Conversation dictionaries with messages
+    """
+    with open(export_path, "r") as f:
+        data = json.load(f)
+
+    for conv in data:
+        yield {
+            "id": conv.get("uuid"),
+            "name": conv.get("name", "Untitled"),
+            "created_at": conv.get("created_at"),
+            "messages": [
+                {
+                    "role": "user" if msg.get("sender") == "human" else "assistant",
+                    "content": msg.get("text", ""),
+                    "created_at": msg.get("created_at"),
+                }
+                for msg in conv.get("chat_messages", [])
+            ],
+        }
+
+
+def is_draft_request(text: str) -> bool:
+    """Detect if user is asking for help drafting text."""
+    patterns = [
+        r"write (me )?a",
+        r"draft (me )?a",
+        r"help me write",
+        r"can you write",
+        r"make (me )?a",
+        r"does this sound",
+        r"is this (good|right|ok)",
+        r"how does this sound",
+        r"rewrite this",
+        r"improve this",
+        r"fix this",
+        r"better way to say",
+    ]
+    text_lower = text.lower()
+    return any(re.search(pattern, text_lower) for pattern in patterns)
+
+
+def is_correction(user_msg: str, prev_assistant_msg: str) -> bool:
+    """
+    Detect if user message is correcting the assistant's draft.
+
+    Patterns:
+    - "No, make it..."
+    - "Actually, change it to..."
+    - "Too formal, use..."
+    - User provides alternative text after assistant's draft
+    """
+    user_lower = user_msg.lower()
+
+    correction_markers = [
+        "no,",
+        "actually,",
+        "instead",
+        "change it",
+        "make it",
+        "too formal",
+        "too long",
+        "too short",
+        "simpler",
+        "more casual",
+        "more professional",
+        "shorter",
+        "just say",
+        "better:",
+        "fix:",
+        "use this:",
+    ]
+
+    # Check for correction markers
+    if any(marker in user_lower for marker in correction_markers):
+        return True
+
+    # Check if user provides alternative text (quoted or after colon)
+    if '"' in user_msg or "'" in user_msg or ":" in user_msg:
+        # User might be providing alternative text
+        return True
+
+    return False
+
+
+def extract_correction_pairs(conversation: dict) -> list[dict]:
+    """
+    Extract (draft, correction) pairs from a conversation.
+
+    Returns:
+        List of correction dictionaries with:
+        - original_draft: AI's draft
+        - user_correction: User's correction/feedback
+        - context: What they were drafting
+    """
+    corrections = []
+    messages = conversation["messages"]
+
+    for i in range(len(messages) - 1):
+        current = messages[i]
+        next_msg = messages[i + 1] if i + 1 < len(messages) else None
+
+        # Look for pattern: User asks for draft → AI provides → User corrects
+        if current["role"] == "user" and is_draft_request(current["content"]):
+            # Find AI's response
+            if next_msg and next_msg["role"] == "assistant":
+                ai_draft = next_msg["content"]
+
+                # Check if user corrects it
+                if i + 2 < len(messages):
+                    user_response = messages[i + 2]
+                    if user_response["role"] == "user" and is_correction(user_response["content"], ai_draft):
+                        corrections.append(
+                            {
+                                "context": current["content"],
+                                "ai_draft": ai_draft,
+                                "user_correction": user_response["content"],
+                                "conversation_name": conversation["name"],
+                            }
+                        )
+
+    return corrections
+
+
+def analyze_correction_patterns(corrections: list[dict]) -> dict:
+    """
+    Analyze what user consistently changes in AI drafts.
+
+    Returns:
+        Dictionary with patterns like:
+        - length_changes: Shorter/longer
+        - formality_changes: More/less formal
+        - common_edits: Specific words/phrases changed
+        - tone_adjustments: Casual/professional
+    """
+    if not corrections:
+        return {}
+
+    patterns = {
+        "total_corrections": len(corrections),
+        "makes_shorter": 0,
+        "makes_longer": 0,
+        "removes_formality": 0,
+        "adds_emojis": 0,
+        "simplifies_language": 0,
+        "common_feedback": [],
+    }
+
+    for corr in corrections:
+        draft = corr["ai_draft"]
+        feedback = corr["user_correction"].lower()
+
+        # Length changes
+        if "shorter" in feedback or "too long" in feedback or "brief" in feedback:
+            patterns["makes_shorter"] += 1
+        if "longer" in feedback or "more detail" in feedback:
+            patterns["makes_longer"] += 1
+
+        # Formality
+        if any(word in feedback for word in ["casual", "informal", "too formal", "stiff"]):
+            patterns["removes_formality"] += 1
+
+        # Emojis
+        if any(char in corr["user_correction"] for char in ["😂", "😊", "🔥", "💪", "👍", "❤️"]):
+            patterns["adds_emojis"] += 1
+
+        # Simplification
+        if any(word in feedback for word in ["simpler", "simple", "easier", "plain"]):
+            patterns["simplifies_language"] += 1
+
+        # Extract common feedback phrases
+        feedback_phrases = [
+            "too formal",
+            "too long",
+            "too short",
+            "more casual",
+            "less formal",
+            "simpler",
+            "add emoji",
+            "remove",
+        ]
+        for phrase in feedback_phrases:
+            if phrase in feedback:
+                patterns["common_feedback"].append(phrase)
+
+    return patterns
+
+
+def extract_user_final_versions(corrections: list[dict]) -> list[str]:
+    """
+    Extract the final versions user actually used/approved.
+
+    These are the gold standard examples of their style.
+    """
+    final_versions = []
+
+    for corr in corrections:
+        # Try to extract quoted text from user's correction
+        user_text = corr["user_correction"]
+
+        # Look for quoted text
+        quoted = re.findall(r'"([^"]+)"', user_text)
+        if quoted:
+            final_versions.extend(quoted)
+
+        # Look for text after colon
+        if ":" in user_text:
+            parts = user_text.split(":", 1)
+            if len(parts) > 1:
+                final_versions.append(parts[1].strip())
+
+    return final_versions

brainlayer/pipeline/extract_markdown.py

@@ -0,0 +1,195 @@
+"""Extract and classify markdown files for indexing."""
+
+import re
+from pathlib import Path
+from typing import Iterator
+
+from .classify import ClassifiedContent, ContentType, ContentValue
+
+
+def find_markdown_files(
+    root: Path, patterns: list[str] | None = None, exclude: list[str] | None = None
+) -> Iterator[Path]:
+    """
+    Find markdown files matching glob patterns.
+
+    Args:
+        root: Root directory to search
+        patterns: Glob patterns to match (default: ["**/*.md"])
+        exclude: Directory names to exclude (default: common non-content dirs)
+
+    Yields:
+        Path objects for matching markdown files
+    """
+    if patterns is None:
+        patterns = ["**/*.md"]
+    if exclude is None:
+        exclude = ["node_modules", ".git", "dist", "__pycache__", ".venv", "venv"]
+
+    exclude_set = set(exclude)
+
+    for pattern in patterns:
+        for path in root.glob(pattern):
+            # Skip excluded directories
+            if any(part in exclude_set for part in path.parts):
+                continue
+            if path.is_file():
+                yield path
+
+
+def parse_markdown(file_path: Path) -> list[dict]:
+    """
+    Parse a markdown file into sections.
+
+    Splits on h2 headers (## ) to create logical chunks.
+    Each section includes header hierarchy for context.
+
+    Returns:
+        List of dicts with keys: content, header, parent_headers, tags
+    """
+    content = file_path.read_text(encoding="utf-8", errors="replace")
+    sections = []
+
+    # Extract frontmatter if present
+    frontmatter = {}
+    if content.startswith("---"):
+        parts = content.split("---", 2)
+        if len(parts) >= 3:
+            # Simple YAML parsing for common keys
+            for line in parts[1].strip().split("\n"):
+                if ":" in line:
+                    key, _, value = line.partition(":")
+                    frontmatter[key.strip()] = value.strip().strip("\"'")
+            content = parts[2]
+
+    # Extract tags from content (#tag patterns not in code blocks)
+    tags = set()
+    for match in re.finditer(r"(?<!\S)#([a-zA-Z][a-zA-Z0-9_-]*)", content):
+        tag = match.group(1).lower()
+        # Skip common markdown/code patterns
+        if tag not in ("include", "define", "ifdef", "endif", "pragma", "import"):
+            tags.add(tag)
+
+    # Split by h2 headers
+    h1_match = re.search(r"^#\s+(.+)$", content, re.MULTILINE)
+    h1_title = h1_match.group(1) if h1_match else file_path.stem
+
+    # Find all h2 sections
+    h2_pattern = r"^##\s+(.+)$"
+    h2_positions = [(m.start(), m.group(1)) for m in re.finditer(h2_pattern, content, re.MULTILINE)]
+
+    if not h2_positions:
+        # No h2 headers - treat entire file as one section
+        sections.append(
+            {
+                "content": content.strip(),
+                "header": h1_title,
+                "parent_headers": [],
+                "tags": list(tags),
+                "frontmatter": frontmatter,
+            }
+        )
+    else:
+        # Content before first h2
+        if h2_positions[0][0] > 0:
+            intro = content[: h2_positions[0][0]].strip()
+            if intro and len(intro) > 50:  # Skip trivial intros
+                sections.append(
+                    {
+                        "content": intro,
+                        "header": h1_title,
+                        "parent_headers": [],
+                        "tags": list(tags),
+                        "frontmatter": frontmatter,
+                    }
+                )
+
+        # Each h2 section
+        for i, (pos, header) in enumerate(h2_positions):
+            end_pos = h2_positions[i + 1][0] if i + 1 < len(h2_positions) else len(content)
+            section_content = content[pos:end_pos].strip()
+
+            sections.append(
+                {
+                    "content": section_content,
+                    "header": header,
+                    "parent_headers": [h1_title],
+                    "tags": list(tags),
+                    "frontmatter": frontmatter,
+                }
+            )
+
+    return sections
+
+
+def classify_by_path(file_path: Path) -> tuple[ContentType, ContentValue]:
+    """
+    Classify a markdown file based on its path.
+
+    Returns:
+        Tuple of (ContentType, ContentValue) based on path patterns
+    """
+    path_str = str(file_path).lower()
+    name = file_path.name.lower()
+
+    # CLAUDE.md files are high-value project config
+    if name == "claude.md":
+        return ContentType.PROJECT_CONFIG, ContentValue.HIGH
+
+    # Learnings are curated high-value content
+    if "/learnings/" in path_str or "learnings/" in path_str:
+        return ContentType.LEARNING, ContentValue.HIGH
+
+    # Skills are high-value reference
+    if "/skills/" in path_str or "skills/" in path_str:
+        return ContentType.SKILL, ContentValue.HIGH
+
+    # Research docs
+    if "/research/" in path_str or "research/" in path_str:
+        return ContentType.RESEARCH, ContentValue.HIGH
+
+    # PRD archives - medium value
+    if "/prd" in path_str or "prd-" in path_str or "prd_" in path_str:
+        return ContentType.PRD_ARCHIVE, ContentValue.MEDIUM
+
+    # Verification rounds - low value (voluminous)
+    if "/verification" in path_str or "verification-" in path_str:
+        return ContentType.VERIFICATION, ContentValue.LOW
+
+    # Default: documentation
+    return ContentType.DOCUMENTATION, ContentValue.MEDIUM
+
+
+def extract_markdown_content(file_path: Path) -> list[ClassifiedContent]:
+    """
+    Extract and classify content from a markdown file.
+
+    Combines parsing and classification into ready-to-chunk content.
+
+    Returns:
+        List of ClassifiedContent objects for each section
+    """
+    content_type, value = classify_by_path(file_path)
+    sections = parse_markdown(file_path)
+
+    results = []
+    for section in sections:
+        # Build context string with header hierarchy
+        header_context = " > ".join(section["parent_headers"] + [section["header"]])
+
+        results.append(
+            ClassifiedContent(
+                content=section["content"],
+                content_type=content_type,
+                value=value,
+                metadata={
+                    "source_file": str(file_path),
+                    "header": section["header"],
+                    "header_context": header_context,
+                    "tags": section["tags"],
+                    "frontmatter": section.get("frontmatter", {}),
+                },
+            )
+        )
+
+    return results