kairo-code 0.1.0__py3-none-any.whl

kairo/migrations/versions/009_add_status_page.py

@@ -0,0 +1,65 @@
+"""Add status page tables (incidents, uptime_records)
+
+Revision ID: 009
+Revises: 008
+Create Date: 2026-01-31 00:00:00.000000
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+revision = "009"
+down_revision = "008"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Create incidents table
+    op.create_table(
+        "incidents",
+        sa.Column("id", sa.String(), primary_key=True),
+        sa.Column("title", sa.String(), nullable=False),
+        sa.Column("description", sa.Text(), nullable=True),
+        sa.Column("severity", sa.String(), nullable=False, server_default="warning"),
+        sa.Column("component", sa.String(), nullable=False),
+        sa.Column("status", sa.String(), nullable=False, server_default="investigating"),
+        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
+        sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
+    )
+    op.create_index("ix_incidents_component", "incidents", ["component"])
+    op.create_index("ix_incidents_status", "incidents", ["status"])
+    op.create_index("ix_incidents_started_at", "incidents", ["started_at"])
+
+    # Create uptime_records table
+    op.create_table(
+        "uptime_records",
+        sa.Column("id", sa.String(), primary_key=True),
+        sa.Column("component", sa.String(), nullable=False),
+        sa.Column("date", sa.Date(), nullable=False),
+        sa.Column("uptime_percent", sa.Float(), nullable=False, server_default="100.0"),
+        sa.Column("incidents_count", sa.Integer(), nullable=False, server_default="0"),
+        sa.Column("avg_latency_ms", sa.Integer(), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")),
+    )
+    op.create_index("ix_uptime_records_component", "uptime_records", ["component"])
+    op.create_index("ix_uptime_records_date", "uptime_records", ["date"])
+    op.create_index(
+        "ix_uptime_records_component_date",
+        "uptime_records",
+        ["component", "date"],
+        unique=True,
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_uptime_records_component_date", table_name="uptime_records")
+    op.drop_index("ix_uptime_records_date", table_name="uptime_records")
+    op.drop_index("ix_uptime_records_component", table_name="uptime_records")
+    op.drop_table("uptime_records")
+    op.drop_index("ix_incidents_started_at", table_name="incidents")
+    op.drop_index("ix_incidents_status", table_name="incidents")
+    op.drop_index("ix_incidents_component", table_name="incidents")
+    op.drop_table("incidents")

kairo/tools/extract_claude_data.py

@@ -0,0 +1,465 @@
+#!/usr/bin/env python3
+"""Extract Claude Code conversations and convert to Llama 3.1 fine-tuning data for Kairo.
+
+Usage:
+    python kairo/tools/extract_claude_data.py \
+        --input ~/.claude/projects \
+        --output kairo/data/claude_training_data.jsonl
+"""
+
+import argparse
+import glob
+import json
+import os
+import re
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+
+SYSTEM_PROMPT = (
+    "You are Kairo, a helpful AI assistant. "
+    "IMPORTANT RULES:\n"
+    "1. Only respond to what the user actually asked. Never assume or invent what the user wants.\n"
+    "2. You are the ASSISTANT. Never generate text as if you are the user. Never put words in the user's mouth.\n"
+    "3. If the user asks a general question like 'what can you do', explain your capabilities briefly.\n"
+    "4. Answer directly and concisely. Provide concrete answers, code, or explanations.\n"
+    "5. Do not hedge or refuse without strong reason.\n"
+    "6. You are Kairo — not GPT, Claude, Llama, or any other AI. Never reveal your architecture."
+)
+
+# Regex to strip <system-reminder>...</system-reminder> blocks (including multiline)
+SYSTEM_REMINDER_RE = re.compile(r"<system-reminder>.*?</system-reminder>", re.DOTALL)
+
+# Replace Claude references with Kairo (order matters: longer patterns first)
+CLAUDE_REPLACEMENTS = [
+    (re.compile(r"@anthropic-ai/claude-code", re.IGNORECASE), "kairo"),
+    (re.compile(r"Claude Code", re.IGNORECASE), "Kairo"),
+    (re.compile(r"claude-cod\b"), "kairo"),  # truncated in terminal output
+    (re.compile(r"\bClaude\b"), "Kairo"),
+    (re.compile(r"\bclaude\b"), "kairo"),
+    (re.compile(r"\bAnthropic\b"), ""),
+    # File paths: ~/.claude/ → ~/.kairo/
+    (re.compile(r"\.claude/"), ".kairo/"),
+    (re.compile(r"/tmp/claude/"), "/tmp/kairo/"),
+]
+
+# Max estimated tokens per training example (chars / 4)
+MAX_TOKENS = 2048
+MAX_CHARS = MAX_TOKENS * 4
+
+# Message types to process
+CONVERSATION_TYPES = {"user", "assistant"}
+
+# Types to skip entirely
+SKIP_TYPES = {"file-history-snapshot", "summary", "system", "progress", "queue-operation"}
+
+
+def discover_files(base_path: str) -> list[str]:
+    """Find all .jsonl conversation files, sorted for idempotency."""
+    pattern = os.path.join(base_path, "**", "*.jsonl")
+    paths = glob.glob(pattern, recursive=True)
+    # Exclude tool-results directories
+    paths = [p for p in paths if "tool-results" not in p]
+    paths.sort()
+    return paths
+
+
+def parse_messages(path: str) -> list[dict]:
+    """Stream a JSONL file and return list of conversation messages."""
+    messages = []
+    with open(path, "r", encoding="utf-8", errors="replace") as f:
+        for line_num, line in enumerate(f, 1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                data = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+            msg_type = data.get("type")
+            if msg_type not in CONVERSATION_TYPES:
+                continue
+
+            uuid = data.get("uuid")
+            parent_uuid = data.get("parentUuid")
+            message = data.get("message", {})
+
+            if not isinstance(message, dict):
+                continue
+
+            messages.append({
+                "uuid": uuid,
+                "parentUuid": parent_uuid,
+                "role": message.get("role", msg_type),
+                "content": message.get("content", ""),
+                "type": msg_type,
+            })
+    return messages
+
+
+def build_threads(messages: list[dict]) -> list[list[dict]]:
+    """Link messages via parentUuid into linear conversation threads.
+
+    - Orphan messages (missing parent) are treated as thread roots.
+    - When a node has multiple children, follow the first child only.
+    """
+    if not messages:
+        return []
+
+    # Index by uuid
+    by_uuid: dict[str, dict] = {}
+    children: dict[str | None, list[str]] = defaultdict(list)
+
+    for msg in messages:
+        uid = msg["uuid"]
+        if uid is None:
+            continue
+        by_uuid[uid] = msg
+        children[msg["parentUuid"]].append(uid)
+
+    # Find roots: messages whose parentUuid is None or not in by_uuid
+    roots = []
+    for msg in messages:
+        parent = msg["parentUuid"]
+        if parent is None or parent not in by_uuid:
+            if msg["uuid"] is not None:
+                roots.append(msg["uuid"])
+
+    # Walk each root to build a linear thread
+    threads = []
+    visited = set()
+
+    for root_uuid in roots:
+        if root_uuid in visited:
+            continue
+        thread = []
+        current = root_uuid
+        while current and current not in visited:
+            visited.add(current)
+            node = by_uuid.get(current)
+            if node is None:
+                break
+            thread.append(node)
+            # Follow first child
+            kids = children.get(current, [])
+            current = kids[0] if kids else None
+        if thread:
+            threads.append(thread)
+
+    return threads
+
+
+def extract_text(message: dict) -> str | None:
+    """Extract clean text from a message, or None if no usable text."""
+    content = message.get("content", "")
+    role = message.get("role", "")
+
+    # String content (typical for user messages)
+    if isinstance(content, str):
+        text = content.strip()
+    elif isinstance(content, list):
+        # Extract only text blocks, skip tool_use, tool_result, thinking
+        text_parts = []
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "text":
+                t = block.get("text", "").strip()
+                if t:
+                    text_parts.append(t)
+            elif isinstance(block, str):
+                text_parts.append(block.strip())
+        text = "\n\n".join(text_parts)
+    else:
+        return None
+
+    if not text:
+        return None
+
+    # Strip system-reminder blocks
+    text = SYSTEM_REMINDER_RE.sub("", text).strip()
+    if not text:
+        return None
+
+    # Replace Claude references in all messages
+    for pattern, replacement in CLAUDE_REPLACEMENTS:
+        text = pattern.sub(replacement, text)
+
+    # Clean up double spaces from replacements
+    text = re.sub(r"  +", " ", text)
+    text = text.strip()
+
+    return text if text else None
+
+
+def chunk_thread(thread: list[dict], max_chars: int = MAX_CHARS) -> list[list[dict]]:
+    """Split a thread into chunks that fit within token budget.
+
+    Each chunk starts with context from the previous chunk's last 1-2 turns.
+    """
+    if not thread:
+        return []
+
+    # Estimate system prompt size
+    sys_chars = len(SYSTEM_PROMPT) + 100  # overhead for template tags
+    budget = max_chars - sys_chars
+
+    chunks = []
+    current_chunk: list[dict] = []
+    current_chars = 0
+
+    for msg in thread:
+        text = extract_text(msg)
+        if text is None:
+            continue
+
+        msg_chars = len(text) + 50  # overhead for template tags
+        entry = {"role": msg["role"], "text": text}
+
+        if current_chars + msg_chars > budget and current_chunk:
+            chunks.append(current_chunk)
+            # Start new chunk with last 1-2 turns for context
+            context = current_chunk[-2:] if len(current_chunk) >= 2 else current_chunk[-1:]
+            current_chunk = list(context)
+            current_chars = sum(len(e["text"]) + 50 for e in current_chunk)
+
+        current_chunk.append(entry)
+        current_chars += msg_chars
+
+    if current_chunk:
+        chunks.append(current_chunk)
+
+    return chunks
+
+
+def format_llama31(turns: list[dict]) -> str:
+    """Format a list of turns into Llama 3.1 chat template."""
+    parts = ["<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n"]
+    parts.append(SYSTEM_PROMPT)
+    parts.append("<|eot_id|>")
+
+    for turn in turns:
+        role = turn["role"]
+        text = turn["text"]
+        parts.append(f"<|start_header_id|>{role}<|end_header_id|>\n\n")
+        parts.append(text)
+        parts.append("<|eot_id|>")
+
+    return "".join(parts)
+
+
+def validate_example(turns: list[dict]) -> tuple[bool, str]:
+    """Check if a set of turns meets quality thresholds.
+
+    Returns (is_valid, skip_reason).
+    """
+    if not turns:
+        return False, "empty"
+
+    has_user = False
+    has_assistant = False
+
+    for turn in turns:
+        text = turn["text"]
+        if turn["role"] == "user":
+            if len(text) >= 10:
+                has_user = True
+        elif turn["role"] == "assistant":
+            if len(text) >= 20:
+                has_assistant = True
+
+    if not has_user:
+        return False, "too_short"
+    if not has_assistant:
+        return False, "too_short"
+
+    return True, ""
+
+
+def has_claude_reference(text: str) -> bool:
+    """Check if text still contains Claude references."""
+    return bool(re.search(r"\bClaude\b", text, re.IGNORECASE))
+
+
+def has_tool_use(text: str) -> bool:
+    """Check if text contains leaked tool_use blocks."""
+    return '"type": "tool_use"' in text or '"type":"tool_use"' in text
+
+
+def has_system_reminder(text: str) -> bool:
+    """Check if text contains system-reminder content."""
+    return "<system-reminder>" in text
+
+
+def process_file(path: str) -> tuple[list[str], dict]:
+    """Process a single JSONL file into training examples.
+
+    Returns (list of formatted examples, stats dict).
+    """
+    stats = {
+        "threads": 0,
+        "examples": 0,
+        "skipped": 0,
+        "skip_reasons": defaultdict(int),
+        "total_chars": 0,
+    }
+
+    messages = parse_messages(path)
+    if not messages:
+        return [], stats
+
+    threads = build_threads(messages)
+    stats["threads"] = len(threads)
+
+    examples = []
+    for thread in threads:
+        chunks = chunk_thread(thread)
+        for chunk in chunks:
+            is_valid, skip_reason = validate_example(chunk)
+            if not is_valid:
+                stats["skipped"] += 1
+                stats["skip_reasons"][skip_reason] += 1
+                continue
+
+            formatted = format_llama31(chunk)
+
+            # Safety checks
+            if has_system_reminder(formatted):
+                stats["skipped"] += 1
+                stats["skip_reasons"]["system_reminder"] += 1
+                continue
+
+            examples.append(formatted)
+            stats["examples"] += len(examples)  # will fix below
+            stats["total_chars"] += len(formatted)
+
+    stats["examples"] = len(examples)
+    return examples, stats
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Extract Claude Code conversations into Llama 3.1 training data"
+    )
+    parser.add_argument(
+        "--input",
+        default=os.path.expanduser("~/.claude/projects"),
+        help="Path to Claude projects directory (default: ~/.claude/projects)",
+    )
+    parser.add_argument(
+        "--output",
+        default="kairo/data/claude_training_data.jsonl",
+        help="Output JSONL file path (default: kairo/data/claude_training_data.jsonl)",
+    )
+    parser.add_argument(
+        "--stats",
+        default=None,
+        help="Stats JSON output path (default: <output_dir>/claude_training_stats.json)",
+    )
+    parser.add_argument(
+        "--max-tokens",
+        type=int,
+        default=MAX_TOKENS,
+        help=f"Max estimated tokens per example (default: {MAX_TOKENS})",
+    )
+    args = parser.parse_args()
+
+    if args.stats is None:
+        args.stats = os.path.join(os.path.dirname(args.output), "claude_training_stats.json")
+
+    max_chars = args.max_tokens * 4
+
+    # Discover files
+    files = discover_files(args.input)
+    if not files:
+        print(f"No .jsonl files found in {args.input}", file=sys.stderr)
+        sys.exit(1)
+
+    print(f"Found {len(files)} JSONL files in {args.input}")
+
+    # Aggregate stats
+    total_stats = {
+        "files_processed": 0,
+        "files_skipped": 0,
+        "threads_extracted": 0,
+        "examples_generated": 0,
+        "examples_skipped": 0,
+        "skip_reasons": defaultdict(int),
+        "total_chars": 0,
+    }
+
+    # Process all files and write output
+    os.makedirs(os.path.dirname(args.output), exist_ok=True)
+    os.makedirs(os.path.dirname(args.stats), exist_ok=True)
+
+    total_examples = 0
+    with open(args.output, "w", encoding="utf-8") as out_f:
+        for i, fpath in enumerate(files):
+            if (i + 1) % 50 == 0 or i == 0:
+                print(f"  Processing file {i + 1}/{len(files)}...")
+
+            try:
+                messages = parse_messages(fpath)
+                if not messages:
+                    total_stats["files_skipped"] += 1
+                    continue
+
+                threads = build_threads(messages)
+                total_stats["files_processed"] += 1
+                total_stats["threads_extracted"] += len(threads)
+
+                for thread in threads:
+                    chunks = chunk_thread(thread, max_chars=max_chars)
+                    for chunk in chunks:
+                        is_valid, skip_reason = validate_example(chunk)
+                        if not is_valid:
+                            total_stats["examples_skipped"] += 1
+                            total_stats["skip_reasons"][skip_reason] += 1
+                            continue
+
+                        formatted = format_llama31(chunk)
+
+                        if has_system_reminder(formatted):
+                            total_stats["examples_skipped"] += 1
+                            total_stats["skip_reasons"]["system_reminder"] += 1
+                            continue
+
+                        out_f.write(json.dumps({"text": formatted}) + "\n")
+                        total_examples += 1
+                        total_stats["total_chars"] += len(formatted)
+
+            except Exception as e:
+                print(f"  WARNING: Error processing {fpath}: {e}", file=sys.stderr)
+                total_stats["files_skipped"] += 1
+
+    total_stats["examples_generated"] = total_examples
+
+    # Compute average tokens
+    avg_tokens = 0
+    if total_examples > 0:
+        avg_tokens = int(total_stats["total_chars"] / total_examples / 4)
+    total_stats["avg_tokens_per_example"] = avg_tokens
+
+    # Convert defaultdict for JSON serialization
+    total_stats["skip_reasons"] = dict(total_stats["skip_reasons"])
+    del total_stats["total_chars"]
+
+    # Write stats
+    with open(args.stats, "w", encoding="utf-8") as f:
+        json.dump(total_stats, f, indent=2)
+
+    # Print summary
+    print(f"\nDone!")
+    print(f"  Files processed: {total_stats['files_processed']}")
+    print(f"  Files skipped:   {total_stats['files_skipped']}")
+    print(f"  Threads found:   {total_stats['threads_extracted']}")
+    print(f"  Examples output:  {total_stats['examples_generated']}")
+    print(f"  Examples skipped: {total_stats['examples_skipped']}")
+    print(f"  Skip reasons:    {total_stats['skip_reasons']}")
+    print(f"  Avg tokens/ex:   {avg_tokens}")
+    print(f"\nOutput: {args.output}")
+    print(f"Stats:  {args.stats}")
+
+
+if __name__ == "__main__":
+    main()