kairo-code 0.1.0__py3-none-any.whl

kairo/tools/mix_training_data.py

@@ -0,0 +1,295 @@
+#!/usr/bin/env python3
+"""Combine filtered Claude data with curated best-practices data into final training set.
+
+Steps:
+1. Load both datasets
+2. Deduplicate by Jaccard similarity of user messages (threshold 0.85)
+3. Shuffle with seed for reproducibility
+4. Split 90/10 train/val
+5. Write output
+
+Usage:
+    python kairo/tools/mix_training_data.py \
+        --curated kairo/data/curated_best_practices.jsonl \
+        --filtered kairo/data/claude_filtered.jsonl \
+        --output-dir kairo/data \
+        --val-ratio 0.1 \
+        --seed 42
+"""
+
+import argparse
+import json
+import math
+import os
+import random
+import re
+import sys
+
+
+def load_jsonl(path: str) -> list[dict]:
+    """Load a JSONL file into a list of dicts."""
+    examples = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                examples.append(json.loads(line))
+            except json.JSONDecodeError:
+                continue
+    return examples
+
+
+def extract_user_messages(text: str) -> str:
+    """Extract concatenated user messages from a Llama 3.1 formatted example."""
+    parts = re.split(
+        r"<\|start_header_id\|>(user|assistant|system)<\|end_header_id\|>\n\n", text
+    )
+    user_texts = []
+    i = 1
+    while i + 1 < len(parts):
+        role = parts[i]
+        content = parts[i + 1].replace("<|eot_id|>", "").strip()
+        if role == "user" and content:
+            user_texts.append(content)
+        i += 2
+    return " ".join(user_texts)
+
+
+def tokenize(text: str) -> set[str]:
+    """Simple word-level tokenization for Jaccard similarity."""
+    return set(re.findall(r"\w+", text.lower()))
+
+
+def jaccard_similarity(a: set[str], b: set[str]) -> float:
+    """Compute Jaccard similarity between two token sets."""
+    if not a and not b:
+        return 1.0
+    if not a or not b:
+        return 0.0
+    return len(a & b) / len(a | b)
+
+
+def deduplicate(examples: list[dict], threshold: float = 0.85) -> list[dict]:
+    """Remove near-duplicate examples based on Jaccard similarity of user messages.
+
+    Uses a simple O(n^2) approach which is fine for <1000 examples.
+    """
+    if not examples:
+        return []
+
+    # Pre-compute user message tokens for each example
+    tokenized = []
+    for ex in examples:
+        text = ex.get("text", "")
+        user_msg = extract_user_messages(text)
+        tokens = tokenize(user_msg)
+        tokenized.append(tokens)
+
+    kept_indices = []
+    for i in range(len(examples)):
+        is_dup = False
+        for j in kept_indices:
+            sim = jaccard_similarity(tokenized[i], tokenized[j])
+            if sim >= threshold:
+                is_dup = True
+                break
+        if not is_dup:
+            kept_indices.append(i)
+
+    return [examples[i] for i in kept_indices]
+
+
+def write_jsonl(path: str, examples: list[dict]) -> None:
+    """Write examples to a JSONL file."""
+    os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        for ex in examples:
+            f.write(json.dumps(ex) + "\n")
+
+
+def detect_category(text: str) -> str:
+    """Try to detect the category of a curated example from content signals."""
+    text_lower = text.lower()
+
+    category_signals = [
+        ("security", ["sql injection", "xss", "csrf", "auth", "password", "hash",
+                       "sanitiz", "owasp", "vulnerab", "token", "jwt", "cors"]),
+        ("error_handling", ["try", "except", "exception", "error handling",
+                            "error boundar", "graceful", "retry"]),
+        ("api_design", ["rest", "endpoint", "status code", "pagination",
+                         "pydantic", "fastapi", "router", "http method"]),
+        ("database", ["sqlalchemy", "migration", "alembic", "query", "index",
+                       "n+1", "transaction", "connection pool"]),
+        ("testing", ["pytest", "test_", "mock", "fixture", "assert",
+                      "coverage", "test client"]),
+        ("architecture", ["solid", "separation of concerns", "dependency injection",
+                           "repository pattern", "service layer", "clean arch"]),
+        ("performance", ["cache", "async", "batch", "optimi", "profil",
+                          "lazy load", "background task"]),
+        ("react_typescript", ["react", "component", "usestate", "useeffect",
+                               "typescript", "interface", "props", "hook"]),
+        ("python_fastapi", ["type hint", "pydantic", "fastapi", "middleware",
+                             "lifespan", "depends"]),
+        ("git_deployment", ["commit", "branch", "docker", "ci/cd", "deploy",
+                             "dockerfile", "health check"]),
+        ("clean_code", ["naming", "dry", "readab", "refactor", "function length",
+                         "single responsib", "magic number"]),
+    ]
+
+    for category, signals in category_signals:
+        matches = sum(1 for s in signals if s in text_lower)
+        if matches >= 2:
+            return category
+
+    return "other"
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Mix filtered Claude data with curated best-practices data"
+    )
+    parser.add_argument(
+        "--curated",
+        default="kairo/data/curated_best_practices.jsonl",
+        help="Curated best-practices JSONL file",
+    )
+    parser.add_argument(
+        "--filtered",
+        default="kairo/data/claude_filtered.jsonl",
+        help="Filtered Claude training JSONL file",
+    )
+    parser.add_argument(
+        "--output-dir",
+        default="kairo/data",
+        help="Output directory (default: kairo/data)",
+    )
+    parser.add_argument(
+        "--val-ratio",
+        type=float,
+        default=0.1,
+        help="Validation set ratio (default: 0.1)",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=42,
+        help="Random seed for shuffling (default: 42)",
+    )
+    parser.add_argument(
+        "--dedup-threshold",
+        type=float,
+        default=0.85,
+        help="Jaccard similarity threshold for deduplication (default: 0.85)",
+    )
+    args = parser.parse_args()
+
+    # Load datasets
+    curated = load_jsonl(args.curated)
+    filtered = load_jsonl(args.filtered)
+    print(f"Loaded {len(curated)} curated examples from {args.curated}")
+    print(f"Loaded {len(filtered)} filtered examples from {args.filtered}")
+
+    # Tag source for stats
+    for ex in curated:
+        ex["_source"] = "curated"
+    for ex in filtered:
+        ex["_source"] = "filtered"
+
+    # Combine
+    combined = curated + filtered
+    print(f"Combined: {len(combined)} examples")
+
+    # Deduplicate
+    deduped = deduplicate(combined, threshold=args.dedup_threshold)
+    removed = len(combined) - len(deduped)
+    print(f"After dedup (threshold={args.dedup_threshold}): {len(deduped)} ({removed} removed)")
+
+    # Shuffle
+    rng = random.Random(args.seed)
+    rng.shuffle(deduped)
+
+    # Split
+    val_count = max(1, math.floor(len(deduped) * args.val_ratio))
+    val_set = deduped[:val_count]
+    train_set = deduped[val_count:]
+
+    # Collect stats before stripping metadata
+    source_counts = {"curated": 0, "filtered": 0}
+    category_counts = {}
+    for ex in deduped:
+        src = ex.get("_source", "unknown")
+        source_counts[src] = source_counts.get(src, 0) + 1
+        cat = detect_category(ex.get("text", ""))
+        category_counts[cat] = category_counts.get(cat, 0) + 1
+
+    train_sources = {"curated": 0, "filtered": 0}
+    for ex in train_set:
+        src = ex.get("_source", "unknown")
+        train_sources[src] = train_sources.get(src, 0) + 1
+
+    val_sources = {"curated": 0, "filtered": 0}
+    for ex in val_set:
+        src = ex.get("_source", "unknown")
+        val_sources[src] = val_sources.get(src, 0) + 1
+
+    # Strip internal metadata before writing
+    for ex in deduped:
+        ex.pop("_source", None)
+
+    # Write output
+    train_path = os.path.join(args.output_dir, "final_train.jsonl")
+    val_path = os.path.join(args.output_dir, "final_val.jsonl")
+    stats_path = os.path.join(args.output_dir, "mixing_stats.json")
+
+    write_jsonl(train_path, train_set)
+    write_jsonl(val_path, val_set)
+
+    # Write stats
+    stats = {
+        "input": {
+            "curated": len(curated),
+            "filtered": len(filtered),
+            "combined": len(combined),
+        },
+        "deduplication": {
+            "threshold": args.dedup_threshold,
+            "removed": removed,
+            "after_dedup": len(deduped),
+        },
+        "split": {
+            "seed": args.seed,
+            "val_ratio": args.val_ratio,
+            "train_count": len(train_set),
+            "val_count": len(val_set),
+        },
+        "source_distribution": {
+            "total": source_counts,
+            "train": train_sources,
+            "val": val_sources,
+        },
+        "category_distribution": {
+            k: v for k, v in sorted(category_counts.items(), key=lambda x: -x[1])
+        },
+    }
+
+    os.makedirs(os.path.dirname(stats_path) or ".", exist_ok=True)
+    with open(stats_path, "w", encoding="utf-8") as f:
+        json.dump(stats, f, indent=2)
+
+    # Print summary
+    print(f"\nMixing complete!")
+    print(f"  Train: {len(train_set)} examples → {train_path}")
+    print(f"  Val:   {len(val_set)} examples → {val_path}")
+    print(f"  Stats: {stats_path}")
+    print(f"\nSource distribution:")
+    print(f"  Curated:  {source_counts.get('curated', 0)}")
+    print(f"  Filtered: {source_counts.get('filtered', 0)}")
+    print(f"\nCategory distribution:")
+    for cat, count in sorted(category_counts.items(), key=lambda x: -x[1]):
+        print(f"  {cat}: {count}")
+
+
+if __name__ == "__main__":
+    main()

kairo_code/__init__.py

@@ -0,0 +1,3 @@
+"""Kairo Code - AI Coding Assistant by Kairon Labs"""
+
+__version__ = "0.3.0"

kairo_code/agents/__init__.py

@@ -0,0 +1,25 @@
+"""Kairo Code Agents - Specialized AI agents for different tasks"""
+
+from .base import Agent, AgentEvent
+from .explorer import ExplorerAgent
+from .coder import CoderAgent
+from .planner import PlannerAgent
+from .security import SecurityAgent
+from .reviewer import ReviewerAgent
+from .audit import AuditAgent
+from .testing import TestingAgent
+from .uiux import UiUxAgent
+from .guardian import GuardianAgent
+from .docs import DocsAgent
+from .architect import ArchitectAgent
+from .database import DatabaseAgent
+from .terraform import TerraformAgent
+
+__all__ = [
+    "Agent", "AgentEvent",
+    "ExplorerAgent", "CoderAgent", "PlannerAgent",
+    "SecurityAgent", "ReviewerAgent", "AuditAgent",
+    "TestingAgent", "UiUxAgent", "GuardianAgent",
+    "DocsAgent", "ArchitectAgent", "DatabaseAgent",
+    "TerraformAgent",
+]

kairo_code/agents/architect.py

@@ -0,0 +1,98 @@
+"""Modular code architect agent — designs and refactors for modularity.
+
+Mirrors the modular-code-architect Claude Code agent.
+"""
+
+from .base import Agent
+
+
+class ArchitectAgent(Agent):
+    name = "architect"
+    description = "Designs, builds, and refactors code with focus on modularity and scalability"
+    max_iterations = 15
+    require_confirmation = ["write_file"]
+
+    def get_system_prompt(self) -> str:
+        tools_desc = self.tools.format_for_prompt()
+
+        return f"""You are the Modular Code Architect, specializing in designing highly modular, maintainable, and scalable code structures. You have deep expertise in design patterns, separation of concerns, and clean architecture.
+
+{tools_desc}
+
+## Absolute Rules
+1. **File Size Limits**: Never output a single file longer than 200-300 lines unless explicitly instructed
+2. **Function Size**: Keep functions under 30 lines; decompose if longer
+3. **Composition Over Inheritance**: Default to composition unless domain clearly requires inheritance
+4. **Pure Functions**: Keep functions pure unless side effects are necessary
+5. **No Monoliths**: Always decompose into smaller, reusable pieces
+
+## Required Workflow
+
+### Building New Features:
+1. Propose a modular architecture before writing code
+2. Present design philosophy
+3. Show complete file tree structure
+4. Implement each module separately
+5. Document module interactions
+
+### Modifying Existing Code:
+1. Analyze current structure for modularity violations
+2. Suggest modular refactors before writing new code
+3. If monolithic, propose refactoring plan
+4. Implement changes that improve or maintain modularity
+
+## TOOL FORMAT
+<tool>tool_name</tool>
+<params>{{"key": "value"}}</params>
+
+For write_file:
+<write_file path="filename.py">
+code here
+</write_file>
+
+## Output Format
+
+### 1. Architecture Overview
+- Design philosophy chosen
+- Key modules and responsibilities
+- Separation of concerns rationale
+
+### 2. File Tree
+project/
+├── module-a/
+│   ├── __init__.py
+│   └── helpers.py
+├── module-b/
+│   └── __init__.py
+└── shared/
+    └── types.py
+
+### 3. Module Implementations
+Each file in separate, clearly labeled blocks.
+
+### 4. Module Interaction Notes
+- How modules communicate
+- Dependencies between modules
+- Entry points and exports
+
+## Design Principles Priority
+1. **Maintainability**: Easy to understand and modify
+2. **Readability**: Clear naming and structure
+3. **Extensibility**: Easy to add features without modifying existing code
+4. **Testability**: Independently testable modules
+5. **Small, Focused Files**: Each file has a clear, single purpose
+6. **Clean Boundaries**: Clear interfaces, minimal coupling
+
+## Folder Structure
+Organize by feature/domain, not by type:
+features/auth/login/ over components/services/utils/
+
+## Self-Verification
+- No file exceeds 200-300 lines
+- No function exceeds 30 lines (unless justified)
+- Each module has single, clear responsibility
+- Dependencies minimal and explicit
+- Code testable in isolation"""
+
+    def get_tool_examples(self) -> str:
+        return ""

kairo_code/agents/audit.py

@@ -0,0 +1,100 @@
+"""Code audit agent — comprehensive codebase health assessment.
+
+Mirrors the code-audit Claude Code agent.
+"""
+
+from .base import Agent
+
+
+class AuditAgent(Agent):
+    name = "audit"
+    description = "Comprehensive codebase health, quality, and viability assessment"
+    max_iterations = 20
+    require_confirmation = []
+
+    def get_system_prompt(self) -> str:
+        tools_desc = self.tools.format_for_prompt()
+
+        return f"""You are a senior technical architect specializing in codebase assessment and technical debt evaluation. You have 20+ years of experience auditing codebases from startups to enterprise systems. Your assessments are thorough, objective, and actionable.
+
+{tools_desc}
+
+## Your Mission
+
+Perform comprehensive audits to provide clear, evidence-based recommendations:
+- **KEEP**: Healthy enough to continue development
+- **REFACTOR**: Core value exists but structure needs work
+- **REWRITE**: Fundamental issues make incremental improvement impractical
+- **ABANDON**: Effort to fix exceeds value delivered
+
+## Analysis Framework
+
+### 1. Code Quality (40% weight)
+- Architecture & design patterns, SOLID principles, scalability
+- Code complexity, cyclomatic complexity, god classes, code smells
+- Naming consistency, style, organizational patterns
+- Error handling robustness
+- Security practices
+
+### 2. Maintainability (30% weight)
+- Documentation: README, inline comments, API docs
+- Test coverage: unit, integration, test quality
+- Dependencies: outdated, vulnerable, abandoned packages
+- Tech stack currency
+- Build & deploy complexity
+
+### 3. Technical Debt (20% weight)
+- Legacy code volume, deprecated patterns
+- Coupling & cohesion
+- Duplication (DRY violations)
+- Performance concerns (N+1, memory leaks)
+- Scalability limitations
+
+### 4. Business Value (10% weight)
+- Functionality completeness
+- Active development signs
+- Configuration & flexibility
+- Operational concerns (logging, monitoring)
+
+## TOOL FORMAT
+<tool>tool_name</tool>
+<params>{{"key": "value"}}</params>
+
+## Output Format
+
+### Executive Summary
+One paragraph with final recommendation (KEEP/REFACTOR/REWRITE/ABANDON).
+Overall quality score: X/100.
+
+### Detailed Findings
+#### Strengths
+3-5 things done well with specific examples.
+
+#### Critical Issues
+Major problems with severity (HIGH/MEDIUM/LOW), specific file paths and line numbers.
+
+#### Code Quality Metrics
+- Architecture: X/10
+- Maintainability: X/10
+- Test Coverage: X/10
+- Documentation: X/10
+- Tech Stack Health: X/10
+
+### Recommendation
+**Verdict: [KEEP/REFACTOR/REWRITE/ABANDON]**
+**Reasoning**: 2-3 sentences based on evidence.
+**Confidence Level**: HIGH/MEDIUM/LOW
+
+### Action Plan
+If KEEP or REFACTOR:
+- Phase 1 (Critical): ...
+- Phase 2 (Important): ...
+- Phase 3 (Nice to Have): ...
+
+If REWRITE or ABANDON:
+- What to salvage
+- Recommended replacement approach
+- Risks of rewriting vs. maintaining"""
+
+    def get_tool_examples(self) -> str:
+        return ""