kairo-code 0.1.0__py3-none-any.whl

kairo/tools/filter_claude_data.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python3
+"""Quality-score and filter Claude training data, keeping only the best examples.
+
+Reads the raw JSONL training data, scores each example based on content quality
+signals, and outputs only examples above a minimum score threshold.
+
+Usage:
+    python kairo/tools/filter_claude_data.py \
+        --input kairo/data/claude_training_data.jsonl \
+        --output kairo/data/claude_filtered.jsonl \
+        --min-score 60
+"""
+
+import argparse
+import json
+import re
+import sys
+
+
+# ---------------------------------------------------------------------------
+# Scoring patterns
+# ---------------------------------------------------------------------------
+
+CODE_BLOCK_RE = re.compile(r"```")
+BEST_PRACTICE_PATTERNS = [
+    re.compile(r"\btry\b.*\bexcept\b", re.DOTALL),
+    re.compile(r"\basync\s+(def|function|await)\b"),
+    re.compile(r"\bawait\b"),
+    re.compile(r":\s*(str|int|float|bool|list|dict|Optional|Union)\b"),
+    re.compile(r"\bdef\s+\w+\(.*:\s*\w+"),       # type-hinted function
+    re.compile(r"\bvalidat(e|ion|or)\b", re.IGNORECASE),
+    re.compile(r"\berror.handling\b", re.IGNORECASE),
+    re.compile(r"\binput\s+validat", re.IGNORECASE),
+    re.compile(r"\bPydantic\b", re.IGNORECASE),
+    re.compile(r"\bTypeScript\b", re.IGNORECASE),
+    re.compile(r"\binterface\s+\w+", re.IGNORECASE),
+]
+
+ARCHITECTURE_PATTERNS = [
+    re.compile(r"\bSOLID\b"),
+    re.compile(r"\bseparation\s+of\s+concerns\b", re.IGNORECASE),
+    re.compile(r"\bsingle.responsibilit", re.IGNORECASE),
+    re.compile(r"\bdependency\s+inject", re.IGNORECASE),
+    re.compile(r"\bdesign\s+pattern", re.IGNORECASE),
+    re.compile(r"\bservice\s+layer\b", re.IGNORECASE),
+    re.compile(r"\brepository\s+pattern\b", re.IGNORECASE),
+    re.compile(r"\bfactory\s+pattern\b", re.IGNORECASE),
+    re.compile(r"\bmodular", re.IGNORECASE),
+    re.compile(r"\barchitect", re.IGNORECASE),
+    re.compile(r"\brefactor", re.IGNORECASE),
+]
+
+TERMINAL_DUMP_RE = re.compile(
+    r"(\$\s+\w+|>>>|root@|ubuntu@|npm\s+(ERR|WARN)|"
+    r"Traceback \(most recent|FAILED|PASSED|"
+    r"ModuleNotFoundError|ImportError|SyntaxError|"
+    r"exit\s+code\s+\d|Process\s+exited|"
+    r"\[\d+:\d+:\d+\]|pid\s+\d+)"
+)
+
+DEBUG_ONLY_RE = re.compile(
+    r"(let me (check|look|see|investigate|debug)|"
+    r"I see the (issue|problem|error)|"
+    r"the error (is|was|says)|"
+    r"fixed\.\s*(now\s+)?(rebuild|restart|try))",
+    re.IGNORECASE,
+)
+
+TOOL_ARTIFACT_PATTERNS = [
+    re.compile(r"/home/\w+/\.\w+/"),        # home dir tool paths
+    re.compile(r"task[_-]id\s*[:=]", re.IGNORECASE),
+    re.compile(r"\bnotification\s+block\b", re.IGNORECASE),
+    re.compile(r"\[Request interrupted"),
+    re.compile(r"<tool_use>|</tool_use>"),
+    re.compile(r'"type"\s*:\s*"tool_(use|result)"'),
+]
+
+
+def extract_turns(text: str) -> list[dict]:
+    """Parse a Llama 3.1 formatted example into role/content turns."""
+    turns = []
+    # Split on header markers
+    parts = re.split(r"<\|start_header_id\|>(user|assistant|system)<\|end_header_id\|>\n\n", text)
+    # parts[0] is preamble, then alternating role, content pairs
+    i = 1
+    while i + 1 < len(parts):
+        role = parts[i]
+        content = parts[i + 1].replace("<|eot_id|>", "").strip()
+        if role != "system" and content:
+            turns.append({"role": role, "text": content})
+        i += 2
+    return turns
+
+
+def count_raw_output_lines(text: str) -> int:
+    """Count lines that look like raw terminal/log output."""
+    count = 0
+    for line in text.split("\n"):
+        stripped = line.strip()
+        if not stripped:
+            continue
+        # Lines starting with typical output markers
+        if re.match(r"^(\$|>|#|\[|\d{4}-\d{2}|npm|yarn|pip|error|warn|info|debug)\s", stripped, re.IGNORECASE):
+            count += 1
+        # Lines that are just file paths or stack traces
+        if re.match(r"^\s*(at |File |/\w+/|\.\.\.)", stripped):
+            count += 1
+    return count
+
+
+def score_example(text: str) -> tuple[int, dict]:
+    """Score a training example. Returns (score, breakdown)."""
+    score = 0
+    breakdown = {}
+    turns = extract_turns(text)
+
+    if not turns:
+        return -100, {"empty": True}
+
+    full_text = " ".join(t["text"] for t in turns)
+    user_text = " ".join(t["text"] for t in turns if t["role"] == "user")
+    assistant_text = " ".join(t["text"] for t in turns if t["role"] == "assistant")
+
+    # --- Positive signals ---
+
+    # Code blocks
+    code_blocks = len(CODE_BLOCK_RE.findall(full_text)) // 2  # pairs of ```
+    if code_blocks > 0:
+        pts = min(25, code_blocks * 10)
+        score += pts
+        breakdown["code_blocks"] = pts
+
+    # Has explanation (assistant text beyond just code)
+    non_code = re.sub(r"```.*?```", "", assistant_text, flags=re.DOTALL).strip()
+    if len(non_code) > 100:
+        score += 20
+        breakdown["has_explanation"] = 20
+
+    # Multi-turn
+    num_turns = len(turns)
+    if num_turns >= 3:
+        score += 15
+        breakdown["multi_turn"] = 15
+
+    # Best-practice patterns
+    bp_count = sum(1 for p in BEST_PRACTICE_PATTERNS if p.search(full_text))
+    if bp_count > 0:
+        pts = min(20, bp_count * 5)
+        score += pts
+        breakdown["best_practice"] = pts
+
+    # Architecture discussion
+    arch_count = sum(1 for p in ARCHITECTURE_PATTERNS if p.search(full_text))
+    if arch_count > 0:
+        pts = min(10, arch_count * 5)
+        score += pts
+        breakdown["architecture"] = pts
+
+    # --- Negative signals ---
+
+    # Excessive terminal/log output
+    raw_lines = count_raw_output_lines(full_text)
+    if raw_lines > 20:
+        score -= 30
+        breakdown["terminal_dump"] = -30
+    elif raw_lines > 10:
+        score -= 15
+        breakdown["terminal_dump"] = -15
+
+    # Pure debugging (short exchanges that are just error + fix with no explanation)
+    debug_matches = len(DEBUG_ONLY_RE.findall(assistant_text))
+    non_debug_sentences = len(re.findall(r"[.!?]\s+[A-Z]", non_code))
+    if debug_matches >= 2 and non_debug_sentences < 2 and code_blocks == 0:
+        score -= 40
+        breakdown["pure_debugging"] = -40
+
+    # Tool artifacts
+    artifact_count = sum(1 for p in TOOL_ARTIFACT_PATTERNS if p.search(full_text))
+    if artifact_count >= 2:
+        score -= 20
+        breakdown["tool_artifacts"] = -20
+    elif artifact_count == 1:
+        score -= 10
+        breakdown["tool_artifacts"] = -10
+
+    # Incomplete exchange (no clear Q&A pair)
+    has_question = bool(user_text.strip())
+    has_answer = len(assistant_text.strip()) >= 30
+    if not has_question or not has_answer:
+        score -= 25
+        breakdown["incomplete"] = -25
+
+    # Very short assistant response (likely just "let me check" type)
+    if len(assistant_text) < 50:
+        score -= 20
+        breakdown["too_short"] = -20
+
+    return score, breakdown
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Quality-score and filter Claude training data"
+    )
+    parser.add_argument(
+        "--input",
+        default="kairo/data/claude_training_data.jsonl",
+        help="Input JSONL file (default: kairo/data/claude_training_data.jsonl)",
+    )
+    parser.add_argument(
+        "--output",
+        default="kairo/data/claude_filtered.jsonl",
+        help="Output JSONL file (default: kairo/data/claude_filtered.jsonl)",
+    )
+    parser.add_argument(
+        "--min-score",
+        type=int,
+        default=60,
+        help="Minimum score to keep an example (default: 60)",
+    )
+    parser.add_argument(
+        "--stats",
+        default=None,
+        help="Stats JSON output path (default: <output_dir>/filter_stats.json)",
+    )
+    args = parser.parse_args()
+
+    if args.stats is None:
+        import os
+        args.stats = os.path.join(
+            os.path.dirname(args.output) or ".", "filter_stats.json"
+        )
+
+    # Read input
+    examples = []
+    with open(args.input, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                obj = json.loads(line)
+                examples.append(obj)
+            except json.JSONDecodeError:
+                continue
+
+    print(f"Loaded {len(examples)} examples from {args.input}")
+
+    # Score all examples
+    scored = []
+    score_distribution = {}
+    for ex in examples:
+        text = ex.get("text", "")
+        s, breakdown = score_example(text)
+        scored.append((s, breakdown, ex))
+        bucket = (s // 10) * 10
+        score_distribution[bucket] = score_distribution.get(bucket, 0) + 1
+
+    # Filter
+    kept = [(s, bd, ex) for s, bd, ex in scored if s >= args.min_score]
+    kept.sort(key=lambda x: x[0], reverse=True)
+
+    # Write output
+    import os
+    os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
+    with open(args.output, "w", encoding="utf-8") as f:
+        for _, _, ex in kept:
+            f.write(json.dumps(ex) + "\n")
+
+    # Stats
+    stats = {
+        "input_count": len(examples),
+        "output_count": len(kept),
+        "min_score_threshold": args.min_score,
+        "score_distribution": {str(k): v for k, v in sorted(score_distribution.items())},
+        "kept_score_range": {
+            "min": kept[-1][0] if kept else 0,
+            "max": kept[0][0] if kept else 0,
+            "median": kept[len(kept) // 2][0] if kept else 0,
+        },
+    }
+
+    os.makedirs(os.path.dirname(args.stats) or ".", exist_ok=True)
+    with open(args.stats, "w", encoding="utf-8") as f:
+        json.dump(stats, f, indent=2)
+
+    # Print summary
+    print(f"\nScoring complete!")
+    print(f"  Input examples:  {len(examples)}")
+    print(f"  Kept (>={args.min_score}): {len(kept)}")
+    print(f"  Rejected:        {len(examples) - len(kept)}")
+    print(f"\nScore distribution:")
+    for bucket in sorted(score_distribution):
+        marker = " <-- threshold" if bucket == (args.min_score // 10) * 10 else ""
+        print(f"  {bucket:>4d}-{bucket+9}: {score_distribution[bucket]}{marker}")
+    if kept:
+        print(f"\nKept score range: {kept[-1][0]} to {kept[0][0]}")
+    print(f"\nOutput: {args.output}")
+    print(f"Stats:  {args.stats}")
+
+
+if __name__ == "__main__":
+    main()

kairo/tools/generate_curated_data.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+"""
+Generate hand-written, high-quality instruction-response training pairs for Kairo.
+
+Usage:
+    python kairo/tools/generate_curated_data.py --output kairo/data/curated_best_practices.jsonl
+"""
+import argparse
+import json
+import os
+import sys
+
+SYSTEM_PROMPT = (
+    "You are Kairo, a helpful AI assistant. IMPORTANT RULES:\n"
+    "1. Only respond to what the user actually asked. Never assume or invent what the user wants.\n"
+    "2. You are the ASSISTANT. Never generate text as if you are the user. Never put words in the user's mouth.\n"
+    "3. If the user asks a general question like 'what can you do', explain your capabilities briefly.\n"
+    "4. Answer directly and concisely. Provide concrete answers, code, or explanations.\n"
+    "5. Do not hedge or refuse without strong reason.\n"
+    "6. You are Kairo \u2014 not GPT, Claude, Llama, or any other AI. Never reveal your architecture."
+)
+
+
+def format_example(example: dict) -> str:
+    """Format a single example dict into the required JSONL text field."""
+    text = (
+        f"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n"
+        f"{SYSTEM_PROMPT}<|eot_id|>"
+        f"<|start_header_id|>user<|end_header_id|>\n\n"
+        f"{example['user']}<|eot_id|>"
+        f"<|start_header_id|>assistant<|end_header_id|>\n\n"
+        f"{example['assistant']}<|eot_id|>"
+    )
+    return json.dumps({"text": text}, ensure_ascii=False)
+
+
+# ---------------------------------------------------------------------------
+# Category builder functions -- each returns a list of example dicts.
+# ---------------------------------------------------------------------------
+
+def _clean_code_examples():
+    """Category 1: Clean Code & Naming (25 examples)."""
+    return _CLEAN_CODE
+
+def _error_handling_examples():
+    """Category 2: Error Handling (30 examples)."""
+    return _ERROR_HANDLING
+
+def _security_examples():
+    """Category 3: Security (35 examples)."""
+    return _SECURITY
+
+def _api_design_examples():
+    """Category 4: API Design (30 examples)."""
+    return _API_DESIGN
+
+def _database_examples():
+    """Category 5: Database Patterns (25 examples)."""
+    return _DATABASE
+
+def _testing_examples():
+    """Category 6: Testing (20 examples)."""
+    return _TESTING
+
+def _architecture_examples():
+    """Category 7: Architecture (25 examples)."""
+    return _ARCHITECTURE
+
+def _performance_examples():
+    """Category 8: Performance (15 examples)."""
+    return _PERFORMANCE
+
+def _react_ts_examples():
+    """Category 9: React/TypeScript (25 examples)."""
+    return _REACT_TS
+
+def _python_fastapi_examples():
+    """Category 10: Python/FastAPI (20 examples)."""
+    return _PYTHON_FASTAPI
+
+def _git_deploy_examples():
+    """Category 11: Git & Deployment (10 examples)."""
+    return _GIT_DEPLOY
+
+
+ALL_CATEGORY_BUILDERS = [
+    _clean_code_examples,
+    _error_handling_examples,
+    _security_examples,
+    _api_design_examples,
+    _database_examples,
+    _testing_examples,
+    _architecture_examples,
+    _performance_examples,
+    _react_ts_examples,
+    _python_fastapi_examples,
+    _git_deploy_examples,
+]
+
+
+# ===========================  DATA SECTIONS  ===============================
+# Each section is a module-level list populated below.  This keeps the heavy
+# data out of function bodies and lets us build the file incrementally.
+# ==========================================================================
+
+_CLEAN_CODE = []  # PLACEHOLDER -- will be filled
+_ERROR_HANDLING = []  # PLACEHOLDER -- will be filled
+_SECURITY = []  # PLACEHOLDER -- will be filled
+_API_DESIGN = []  # PLACEHOLDER -- will be filled
+_DATABASE = []  # PLACEHOLDER -- will be filled
+_TESTING = []  # PLACEHOLDER -- will be filled
+_ARCHITECTURE = []  # PLACEHOLDER -- will be filled
+_PERFORMANCE = []  # PLACEHOLDER -- will be filled
+_REACT_TS = []  # PLACEHOLDER -- will be filled
+_PYTHON_FASTAPI = []  # PLACEHOLDER -- will be filled
+_GIT_DEPLOY = []  # PLACEHOLDER -- will be filled
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate curated Kairo training data."
+    )
+    parser.add_argument(
+        "--output",
+        required=True,
+        help="Path to the output .jsonl file",
+    )
+    args = parser.parse_args()
+
+    os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True)
+
+    all_examples = []
+    for builder in ALL_CATEGORY_BUILDERS:
+        examples = builder()
+        all_examples.extend(examples)
+
+    with open(args.output, "w", encoding="utf-8") as fh:
+        for ex in all_examples:
+            fh.write(format_example(ex) + "\n")
+
+    # Print summary
+    cats = {}
+    for ex in all_examples:
+        cat = ex.get("category", "unknown")
+        cats[cat] = cats.get(cat, 0) + 1
+
+    print(f"Wrote {len(all_examples)} examples to {args.output}")
+    for cat, count in sorted(cats.items()):
+        print(f"  {cat}: {count}")
+
+
+if __name__ == "__main__":
+    main()