thread-archive 0.0.2__py3-none-any.whl

thread_archive/_thread_import/parsers/claude_code_blocks.py

@@ -0,0 +1,415 @@
+"""Pure content-block / message-list helpers for the Claude Code parser.
+
+These are the stateless parsing helpers extracted out of
+``ClaudeCodeParser`` (see ``claude_code.py``). They take their inputs
+explicitly and call only the shared static helpers on ``ProviderParser``
+(``create_text_block``), so they carry no instance state. ``ClaudeCodeParser``
+keeps thin delegating methods that call into here, preserving the class
+surface (interface + test-called private methods) byte-for-byte.
+
+Behavior is identical to the in-class originals: every emitted dict / list /
+content-block shape and return value is unchanged.
+"""
+
+import re
+import uuid as _uuid
+from typing import Any, Dict, List, Tuple, cast
+
+from .base import ContentBlock, NormalizedMessage, ProviderParser
+from .claude_code_ide import _extract_ide_context
+
+# Regex for parsing XML-format tool calls (the `<function_calls>` form),
+# co-located with the XML parser that uses them and also exposed as class
+# attributes on ClaudeCodeParser.
+FUNCTION_CALLS_SPLIT = re.compile(
+    r'(<function_calls>.*?</function_calls>)', re.DOTALL
+)
+INVOKE_PATTERN = re.compile(
+    r'<invoke\s+name="([^"]+)">(.*?)</invoke>', re.DOTALL
+)
+PARAMETER_PATTERN = re.compile(
+    r'<parameter\s+name="([^"]+)">(.*?)</parameter>', re.DOTALL
+)
+
+
+def extract_tool_result_text(content: Any) -> str:
+    """Extract text from tool result content."""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        texts = []
+        for item in content:
+            if isinstance(item, str):
+                texts.append(item)
+            elif isinstance(item, dict):
+                text = item.get("text", "")
+                if text:
+                    texts.append(text)
+        return "\n".join(texts)
+    return ""
+
+
+def user_content_blocks_from_list(
+    content: list,
+) -> "tuple[List[ContentBlock], str, List[Dict[str, Any]]]":
+    """Build content blocks from a user message's list content.
+
+    Returns ``(content_blocks, content_text, ide_context_blocks)``. Handles
+    the tool_result / image / text block shapes; text blocks also yield IDE
+    context that the caller appends (renumbered) after the main blocks.
+    """
+    content_blocks: List[ContentBlock] = []
+    content_text = ""
+    ide_context_blocks: List[Dict[str, Any]] = []
+    # Array of content blocks (tool results, text blocks)
+    seq = 0
+    for block in content:
+        if isinstance(block, dict):
+            block_type = block.get("type", "")
+            if block_type == "tool_result":
+                tool_content = block.get("content", [])
+                # Extract text from tool result content
+                result_text = extract_tool_result_text(tool_content)
+                content_blocks.append(cast(ContentBlock, {
+                    "type": "tool_result",
+                    "tool_use_id": block.get("tool_use_id"),
+                    "content": tool_content,
+                    "text": result_text,
+                    "is_error": block.get("is_error", False),
+                    "seq": seq,
+                }))
+                seq += 1
+            elif block_type == "image":
+                # Preserve image blocks (base64-encoded screenshots, etc.)
+                content_blocks.append(cast(ContentBlock, {
+                    "type": "image",
+                    "source": block.get("source", {}),
+                    "seq": seq,
+                }))
+                seq += 1
+            elif block_type == "text":
+                text = block.get("text", "")
+                if text:
+                    # Extract IDE context from text blocks
+                    cleaned_text, extracted_ide = _extract_ide_context(text)
+                    ide_context_blocks.extend(extracted_ide)
+
+                    if cleaned_text:
+                        content_text = cleaned_text if not content_text else f"{content_text}\n{cleaned_text}"
+                        content_blocks.append(ProviderParser.create_text_block(cleaned_text, seq))
+                        seq += 1
+            elif block_type:
+                # Preserve unknown user block types — "Archivist, Not Filter"
+                # (mirrors append_assistant_block). document / redacted_thinking /
+                # mcp_tool_use / future kinds keep their own block instead of
+                # vanishing from content_blocks (they'd otherwise survive only in
+                # the raw provider_data.line blob).
+                content_blocks.append(cast(ContentBlock, {
+                    "type": block_type,
+                    "raw": block,
+                    "seq": seq,
+                }))
+                seq += 1
+        elif isinstance(block, str):
+            # A bare string inside a user content list — keep it as text rather
+            # than skip it.
+            if block:
+                content_text = block if not content_text else f"{content_text}\n{block}"
+                content_blocks.append(ProviderParser.create_text_block(block, seq))
+                seq += 1
+    return content_blocks, content_text, ide_context_blocks
+
+
+def parse_xml_function_calls(
+    raw_content: str, seq: int
+) -> Tuple[List[Dict[str, Any]], List[str], int]:
+    """Parse old-format <function_calls> XML from string content.
+
+    Old Claude Code sessions (pre-2026) stored tool calls as XML text
+    within the assistant response string. This splits the string into
+    interleaved text and tool_use content blocks.
+
+    Returns:
+        (content_blocks, content_text_parts, updated_seq)
+    """
+    content_blocks: List[Dict[str, Any]] = []
+    content_text_parts: List[str] = []
+
+    segments = FUNCTION_CALLS_SPLIT.split(raw_content)
+
+    for segment in segments:
+        if not segment.strip():
+            continue
+
+        if segment.startswith('<function_calls>'):
+            # Parse each <invoke> within the block
+            for match in INVOKE_PATTERN.finditer(segment):
+                tool_name = match.group(1)
+                invoke_body = match.group(2)
+
+                # Extract parameters
+                input_data = {}
+                for param in PARAMETER_PATTERN.finditer(invoke_body):
+                    input_data[param.group(1)] = param.group(2)
+
+                # Deterministic ID from content
+                tool_call_id = str(_uuid.uuid5(
+                    _uuid.NAMESPACE_URL,
+                    f"{tool_name}:{seq}:{list(input_data.values())[:1]}"
+                ))
+
+                content_blocks.append({
+                    "type": "tool_use",
+                    "tool_call_id": tool_call_id,
+                    "name": tool_name,
+                    "input": input_data,
+                    "seq": seq,
+                })
+                seq += 1
+        else:
+            # Text segment
+            text = segment.strip()
+            if text:
+                content_text_parts.append(text)
+                content_blocks.append(cast(Dict[str, Any], ProviderParser.create_text_block(text, seq)))
+                seq += 1
+
+    return content_blocks, content_text_parts, seq
+
+
+def append_assistant_block(
+    block: Dict[str, Any],
+    content_blocks: List[ContentBlock],
+    content_text_parts: List[str],
+    seq: int,
+) -> int:
+    """Append one list-shaped assistant block; return the next seq value."""
+    block_type = block.get("type", "")
+
+    if block_type == "thinking":
+        thinking_text = block.get("thinking", "")
+        if thinking_text:
+            content_blocks.append(cast(ContentBlock, {
+                "type": "thinking",
+                "text": thinking_text,
+                "signature": block.get("signature"),
+                "seq": seq,
+            }))
+            return seq + 1
+
+    elif block_type == "text":
+        text = block.get("text", "")
+        if text:
+            content_text_parts.append(text)
+            content_blocks.append(ProviderParser.create_text_block(text, seq))
+            return seq + 1
+
+    elif block_type == "tool_use":
+        content_blocks.append({
+            "type": "tool_use",
+            "tool_call_id": block.get("id"),
+            "name": block.get("name", "unknown"),
+            "input": block.get("input", {}),
+            "seq": seq,
+        })
+        return seq + 1
+
+    elif block_type:
+        # Preserve unknown block types - "Archivist, Not Filter"
+        content_blocks.append(cast(ContentBlock, {
+            "type": block_type,
+            "raw": block,
+            "seq": seq,
+        }))
+        return seq + 1
+
+    return seq
+
+
+def assistant_content(
+    raw_content: Any,
+) -> Tuple[List[ContentBlock], List[str]]:
+    """Build (content_blocks, content_text_parts) for an assistant message.
+
+    Handles both the list-of-blocks shape (thinking/text/tool_use/unknown)
+    and the legacy string shape (plain or old-format <function_calls> XML).
+    """
+    content_blocks: List[ContentBlock] = []
+    content_text_parts: List[str] = []
+    seq = 0
+
+    if isinstance(raw_content, list):
+        for block in raw_content:
+            if not isinstance(block, dict):
+                continue
+            seq = append_assistant_block(
+                block, content_blocks, content_text_parts, seq
+            )
+    elif isinstance(raw_content, str):
+        if '<function_calls>' in raw_content:
+            blocks, text_parts, seq = parse_xml_function_calls(
+                raw_content, seq
+            )
+            content_blocks.extend(cast(List[ContentBlock], blocks))
+            content_text_parts.extend(text_parts)
+        else:
+            content_text_parts.append(raw_content)
+            content_blocks.append(ProviderParser.create_text_block(raw_content, 0))
+
+    return content_blocks, content_text_parts
+
+
+def dedup_compaction_replays(
+    messages: List[NormalizedMessage],
+) -> List[NormalizedMessage]:
+    """Drop compaction-replay duplicates.
+
+    Claude Code context compaction replays earlier messages in the JSONL
+    with new UUIDs but identical content+timestamps. Keep the first
+    occurrence of each (role, content_text, created_at) for user/assistant.
+    """
+    seen: set[tuple[str, str, str | None, str | None]] = set()
+    deduped: List[NormalizedMessage] = []
+    for msg in messages:
+        role = msg.get("role", "")
+        if role in ("user", "assistant"):
+            content_text = msg.get("content_text", "")
+            created_at = msg.get("created_at")
+            # A `/model` switch has empty content_text and shares its timestamp with the
+            # command line beside it — so include the switch target in the key, else the
+            # marker-bearing turn collides with an empty one and is wrongly deduped away.
+            model_change = next(
+                (b.get("to_model") for b in msg.get("content_blocks", [])
+                 if isinstance(b, dict) and b.get("type") == "model_change"),
+                None,
+            )
+            key = (role, content_text, str(created_at) if created_at else None, model_change)
+            if key in seen:
+                continue
+            seen.add(key)
+        deduped.append(msg)
+    return deduped
+
+
+def apply_derived_title(messages: List[NormalizedMessage]) -> None:
+    """Derive a conversation title from the first user message and apply it.
+
+    ChatGPT has explicit titles; Claude Code doesn't, so derive from content
+    (first line, max 100 chars, ellipsised when truncated). Mutates each
+    message's ``conversation_title`` in place.
+    """
+    first_user_content = None
+    for msg in messages:
+        if msg.get("role") == "user" and msg.get("content_text"):
+            first_user_content = msg.get("content_text")
+            break
+
+    if first_user_content:
+        # Truncate to reasonable title length (first line, max 100 chars)
+        title = first_user_content.split("\n")[0][:100]
+        if len(first_user_content) > 100 or "\n" in first_user_content:
+            title = title.rstrip() + "..."
+        # Update all messages with the conversation title
+        for msg in messages:
+            msg["conversation_title"] = title
+
+
+def build_chain_indices(
+    messages: List[NormalizedMessage],
+) -> Tuple[Dict[str, NormalizedMessage], Dict[str, List[NormalizedMessage]]]:
+    """Build (by_id, children_of) indices over the message list."""
+    by_id: Dict[str, NormalizedMessage] = {}
+    for m in messages:
+        mid = m.get("provider_message_id")
+        if mid:
+            by_id[mid] = m
+
+    children_of: Dict[str, List[NormalizedMessage]] = {}
+    for m in messages:
+        parent_id = m.get("provider_parent_id")
+        if parent_id:
+            if parent_id not in children_of:
+                children_of[parent_id] = []
+            children_of[parent_id].append(m)
+
+    return by_id, children_of
+
+
+def find_chain_roots(
+    messages: List[NormalizedMessage],
+    by_id: Dict[str, NormalizedMessage],
+) -> List[NormalizedMessage]:
+    """Find chain roots: assistant messages whose parent is NOT an assistant
+    (i.e., parent is user, system, or doesn't exist)."""
+    chain_roots = []
+    for m in messages:
+        if m.get("role") != "assistant":
+            continue
+        parent_id = m.get("provider_parent_id")
+        if not parent_id:
+            chain_roots.append(m)
+        elif parent_id in by_id:
+            parent = by_id[parent_id]
+            if parent.get("role") != "assistant":
+                chain_roots.append(m)
+        else:
+            # Parent not in this session, treat as root
+            chain_roots.append(m)
+    return chain_roots
+
+
+def merge_chain(
+    root: NormalizedMessage,
+    children_of: Dict[str, List[NormalizedMessage]],
+    to_remove: set,
+) -> None:
+    """BFS-collect all descendant assistant blocks into ``root`` in place,
+    marking merged children in ``to_remove``."""
+    root_id = root.get("provider_message_id")
+    if not root_id or root_id in to_remove:
+        return
+
+    all_blocks = list(root.get("content_blocks", []))
+    queue = children_of.get(root_id, [])[:]
+
+    while queue:
+        child = queue.pop(0)
+        if child.get("role") != "assistant":
+            continue
+        child_id = child.get("provider_message_id")
+        if child_id in to_remove:
+            continue
+
+        # Add this child's blocks
+        all_blocks.extend(child.get("content_blocks", []))
+        to_remove.add(child_id)
+
+        # Add this child's children to queue
+        queue.extend(children_of.get(cast(str, child_id), []))
+
+    # Update root's blocks
+    root["content_blocks"] = all_blocks
+
+
+def coalesce_assistant_chains(
+    messages: List[NormalizedMessage],
+) -> List[NormalizedMessage]:
+    """Coalesce linked assistant messages into single messages.
+
+    In Claude Code JSONL, one assistant "turn" is stored as multiple lines
+    linked by parentUuid (thinking -> text -> tool_use -> tool_use...).
+    This merges them into a single message with combined content_blocks.
+    """
+    by_id, children_of = build_chain_indices(messages)
+    chain_roots = find_chain_roots(messages, by_id)
+
+    # For each chain root, collect all descendant assistant messages
+    to_remove: set = set()
+    for root in chain_roots:
+        merge_chain(root, children_of, to_remove)
+
+    # Return messages with merged ones removed
+    return [
+        m for m in messages
+        if m.get("provider_message_id") not in to_remove
+    ]

thread_archive/_thread_import/parsers/claude_code_ide.py

@@ -0,0 +1,122 @@
+"""IDE-context extraction and timestamp helpers for the Claude Code parser.
+
+Stateless, pure helpers shared by the Claude Code parser modules: the
+IDE-context tag extraction (opened files, selections) plus the small
+timestamp-normalization helpers. They take their inputs explicitly and carry
+no instance state. ``claude_code.py`` re-imports them so ``parent.<name>``
+access and import sites resolve to the same objects;
+``claude_code_blocks.py`` imports ``_extract_ide_context`` from here at
+module level (no deferred import — there is no circular-import cycle).
+"""
+
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+from thread_archive._thread_import.timestamps import parse_timestamp, parse_timestamp_iso
+
+
+def _parse_iso_timestamp(ts: Any) -> Optional[str]:
+    """Parse ISO timestamp to normalized format."""
+    return parse_timestamp_iso(ts)
+
+
+def _timestamp_to_order(ts: str) -> int:
+    """Convert ISO timestamp to microsecond-precision order value."""
+    dt = parse_timestamp(ts)
+    return int(dt.timestamp() * 1_000_000) if dt else 0
+
+
+# Regex patterns for IDE context tags
+_IDE_OPENED_FILE_PATTERN = re.compile(
+    r'<ide_opened_file>(.*?)</ide_opened_file>',
+    re.DOTALL
+)
+_IDE_SELECTION_PATTERN = re.compile(
+    r'<ide_selection>(.*?)</ide_selection>',
+    re.DOTALL
+)
+
+# Regex patterns for command context tags (local command execution)
+_COMMAND_TAGS_PATTERN = re.compile(
+    r'<(command-name|command-message|command-args|local-command-stdout|local-command-caveat|local-command-stderr)>(.*?)</\1>',
+    re.DOTALL
+)
+# Generic pattern for other system/context tags to strip from display
+_SYSTEM_CONTEXT_PATTERN = re.compile(
+    r'<(system-reminder|file_contents|context|selection|environment_details)>(.*?)</\1>',
+    re.DOTALL
+)
+
+
+# A manual model switch: Claude Code runs `/model X` and records the confirmation as
+# `<local-command-stdout>Set model to X</local-command-stdout>` on a user line whose
+# command tags are otherwise stripped to nothing (and the turn dropped). Capture the
+# target so the archive can show the same "Switched to X" marker the CLI does.
+_MODEL_SET_PATTERN = re.compile(
+    r'<local-command-stdout>\s*Set model to\s+(.+?)\s*</local-command-stdout>',
+    re.IGNORECASE | re.DOTALL,
+)
+
+
+def _extract_model_change(text: str) -> Optional[str]:
+    """Return the model a ``/model`` command switched to, if ``text`` is that command's
+    ``<local-command-stdout>Set model to X</local-command-stdout>`` confirmation; else None."""
+    if not text:
+        return None
+    m = _MODEL_SET_PATTERN.search(text)
+    return m.group(1).strip() if m else None
+
+
+def _extract_ide_context(text: str) -> Tuple[str, List[Dict[str, Any]]]:
+    """Extract IDE context tags from message text.
+
+    Claude Code embeds IDE context like opened files and selections as XML-like
+    tags in user messages. This function extracts them as structured blocks
+    and returns the cleaned text.
+
+    Returns:
+        Tuple of (cleaned_text, list of ide_context blocks)
+    """
+    ide_blocks: List[Dict[str, Any]] = []
+    seq = 0
+
+    # Extract <ide_opened_file> tags
+    for match in _IDE_OPENED_FILE_PATTERN.finditer(text):
+        content = match.group(1).strip()
+        # Parse the content - typically "The user opened the file {path} in the IDE..."
+        file_path = None
+        if "opened the file " in content:
+            # Extract path between "opened the file " and " in the IDE"
+            path_match = re.search(r'opened the file ([^\s]+)', content)
+            if path_match:
+                file_path = path_match.group(1)
+
+        ide_blocks.append({
+            "type": "ide_context",
+            "context_type": "opened_file",
+            "file_path": file_path,
+            "raw_content": content,
+            "seq": seq,
+        })
+        seq += 1
+
+    # Extract <ide_selection> tags
+    for match in _IDE_SELECTION_PATTERN.finditer(text):
+        content = match.group(1).strip()
+        ide_blocks.append({
+            "type": "ide_context",
+            "context_type": "selection",
+            "raw_content": content,
+            "seq": seq,
+        })
+        seq += 1
+
+    # Remove the tags from the text
+    cleaned = _IDE_OPENED_FILE_PATTERN.sub('', text)
+    cleaned = _IDE_SELECTION_PATTERN.sub('', cleaned)
+    # Also strip command context tags and system context tags
+    cleaned = _COMMAND_TAGS_PATTERN.sub('', cleaned)
+    cleaned = _SYSTEM_CONTEXT_PATTERN.sub('', cleaned)
+    cleaned = cleaned.strip()
+
+    return cleaned, ide_blocks

thread_archive/_thread_import/parsers/claude_code_sessions.py

@@ -0,0 +1,90 @@
+"""Filesystem discovery and loading of Claude Code session files.
+
+These two helpers locate session JSONL files under ``~/.claude/projects/`` and
+read one into a lines/parse-errors dict. They are independent of
+``ClaudeCodeParser`` (no class state) and are re-exported by ``claude_code`` so
+callers can reach them as ``claude_code.<name>``.
+"""
+
+import json
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+
+def find_claude_code_sessions(
+    base_path: Optional[str] = None,
+    include_subagents: bool = True,
+) -> List[Path]:
+    """
+    Find all Claude Code session files.
+
+    Args:
+        base_path: Optional base path. Defaults to ~/.claude/projects/
+        include_subagents: Whether to include agent-*.jsonl files (default: True)
+
+    Returns:
+        List of paths to session JSONL files
+    """
+    if base_path:
+        base = Path(base_path).expanduser()
+    else:
+        base = Path.home() / ".claude" / "projects"
+
+    if not base.exists():
+        return []
+
+    sessions = []
+    for project_dir in base.iterdir():
+        if project_dir.is_dir():
+            for session_file in project_dir.glob("*.jsonl"):
+                is_subagent = session_file.name.startswith("agent-")
+                if is_subagent and not include_subagents:
+                    continue
+                sessions.append(session_file)
+
+    return sorted(sessions, key=lambda p: p.stat().st_mtime, reverse=True)
+
+
+def load_claude_code_session(path: Path) -> Dict[str, Any]:
+    """
+    Load a Claude Code session file.
+
+    Args:
+        path: Path to the session JSONL file
+
+    Returns:
+        Dict with session_id, lines, parse_errors, and is_subagent flag
+    """
+    lines = []
+    parse_errors = []
+    session_id = path.stem  # Use filename without extension as session ID
+    is_subagent = path.name.startswith("agent-")
+
+    # Extract parent session ID from agent filename if applicable
+    # Format: agent-{parent_session_id}.jsonl
+    parent_session_id = None
+    if is_subagent and session_id.startswith("agent-"):
+        parent_session_id = session_id[6:]  # Remove "agent-" prefix
+
+    with open(path, "r", encoding="utf-8") as f:
+        for line_num, line in enumerate(f, start=1):
+            line = line.strip()
+            if line:
+                try:
+                    lines.append(json.loads(line))
+                except json.JSONDecodeError as e:
+                    # Store parse error instead of dropping
+                    parse_errors.append({
+                        "line_number": line_num,
+                        "raw_text": line[:1000],  # Truncate very long lines
+                        "error": str(e),
+                    })
+
+    return {
+        "session_id": session_id,
+        "path": str(path),
+        "lines": lines,
+        "parse_errors": parse_errors,
+        "is_subagent": is_subagent,
+        "parent_session_id": parent_session_id,
+    }

thread_archive/_thread_import/parsers/config/__init__.py

@@ -0,0 +1,26 @@
+"""
+Provider configuration definitions.
+
+Each provider has specific characteristics that affect parsing and validation.
+ProviderConfig captures these in a type-safe, declarative way.
+"""
+
+from .base import (
+    CHATGPT_CONFIG,
+    CLAUDE_CODE_CONFIG,
+    CLAUDE_CONFIG,
+    CURSOR_CONFIG,
+    ProviderConfig,
+    ThinkingExpectation,
+    get_provider_config,
+)
+
+__all__ = [
+    "ProviderConfig",
+    "ThinkingExpectation",
+    "CHATGPT_CONFIG",
+    "CLAUDE_CONFIG",
+    "CLAUDE_CODE_CONFIG",
+    "CURSOR_CONFIG",
+    "get_provider_config",
+]