kite-agent 0.1.0__py3-none-any.whl

kite/observers.py

@@ -0,0 +1,107 @@
+import json
+import os
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+class EventFileLogger:
+    """Standard JSON file logger for EventBus events."""
+    def __init__(self, trace_file: str):
+        if not os.path.isabs(trace_file):
+            trace_file = os.path.join(os.getcwd(), trace_file)
+        self.trace_file = trace_file
+        # Initialize file as an empty list
+        with open(self.trace_file, "w") as f:
+            f.write("[\n]")
+    
+    def on_event(self, event: str, data: Any):
+        """Append event to the JSON list file."""
+        try:
+            # Atomic-ish append to JSON array
+            with open(self.trace_file, "rb+") as f:
+                f.seek(-2, os.SEEK_END)
+                pos = f.tell()
+                f.truncate()
+                if pos > 2: f.write(b",\n")
+                
+                log_entry = {
+                    "timestamp": datetime.now().isoformat(),
+                    "event": event,
+                    "data": self._sanitize(data)
+                }
+                json_str = json.dumps(log_entry, indent=4)
+                f.write(json_str.encode('utf-8'))
+                f.write(b"\n]")
+        except Exception:
+            pass
+
+    def _sanitize(self, data):
+        if isinstance(data, dict):
+            return {k: self._sanitize(v) for k, v in data.items()}
+        elif isinstance(data, list):
+            return [self._sanitize(v) for v in data]
+        elif isinstance(data, (str, int, float, bool, type(None))):
+            return data
+        else:
+            return str(data)
+
+class StateTracker:
+    """Standard run state tracker that persists to JSON."""
+    def __init__(self, session_file: str, event_map: Dict[str, str] = None):
+        if not os.path.isabs(session_file):
+            session_file = os.path.join(os.getcwd(), session_file)
+        self.session_file = session_file
+        # Default map for common events
+        self.event_map = event_map or {
+            "pipeline:lead_result": "leads",
+            "agent:complete": "results"
+        }
+        self.data = {
+            "start_time": datetime.now().isoformat(),
+            "status": "running"
+        }
+        # Initialize collections based on event_map
+        for key in self.event_map.values():
+            if key not in self.data:
+                self.data[key] = []
+
+    def on_event(self, event: str, data: Any):
+        if event in self.event_map:
+            collection_name = self.event_map[event]
+            if collection_name in self.data:
+                # Deduplicate if data has 'name' or 'id'
+                if isinstance(data, dict) and "name" in data:
+                    if any(item.get("name") == data["name"] for item in self.data[collection_name]):
+                        return
+                
+                self.data[collection_name].append(data)
+                self.save()
+        elif event == "pipeline:complete":
+            self.data["status"] = "completed"
+            self.data["end_time"] = datetime.now().isoformat()
+            self.save()
+
+    def save(self):
+        try:
+            with open(self.session_file, "w") as f:
+                json.dump(self.data, f, indent=4)
+        except Exception:
+            pass
+
+class MarkdownReporter:
+    """Standard markdown report generator for real-time updates."""
+    def __init__(self, output_file: str, title: str = "Execution Report"):
+        if not os.path.isabs(output_file):
+            output_file = os.path.join(os.getcwd(), output_file)
+        self.output_file = output_file
+        with open(self.output_file, "w") as f:
+            f.write(f"# {title}\n")
+            f.write(f"*Started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}*\n\n")
+            f.write("Standardized framework report.\n\n---\n\n")
+
+    def append(self, content: str):
+        """Append a section to the report."""
+        try:
+            with open(self.output_file, "a") as f:
+                f.write(content + "\n\n")
+        except Exception:
+            pass

kite/optimization/__init__.py

@@ -0,0 +1,9 @@
+"""
+Optimization Module
+==================
+Resource-aware routing and optimization utilities.
+"""
+
+from .resource_router import ResourceAwareRouter
+
+__all__ = ['ResourceAwareRouter']

kite/optimization/resource_router.py

@@ -0,0 +1,80 @@
+"""
+Resource-Aware Optimization (Chapter 16)
+Dynamically selects the optimal model (resource) based on task complexity.
+"""
+
+from typing import Dict, Any, Optional
+import os
+
+class ResourceAwareRouter:
+    """
+    Routes queries to the most cost-effective model.
+    """
+    def __init__(self, config: Dict[str, Any] = None):
+        config = config or {}
+        
+        # Priority: Config > Env Var > Default Placeholder
+        # Priority: Config > Env Var (Split) > Env Var (Legacy)
+        
+        # 1. Fast Model Resolution
+        fast_provider = config.get("fast_llm_provider") or os.getenv("FAST_LLM_PROVIDER")
+        fast_model_name = config.get("fast_llm_model") or os.getenv("FAST_LLM_MODEL")
+        
+        if fast_provider and fast_model_name:
+            self.fast_model = f"{fast_provider}/{fast_model_name}"
+        else:
+            # Legacy fallback: check for full string in FAST_LLM_MODEL (e.g. "groq/llama...")
+            self.fast_model = config.get("fast_model") or os.getenv("FAST_LLM_MODEL")
+        self.smart_model = config.get("smart_model") or os.getenv("SMART_LLM_MODEL")
+        
+        # Fallback to defaults only if absolutely necessary, but log warning
+        if not self.fast_model:
+            raise ValueError("Configuration Error: 'fast_model' not found. Set FAST_LLM_MODEL env var or pass in config.")
+            
+        if not self.smart_model:
+            # Fallback to main LLM from env if strictly necessary (User request)
+            main_provider = os.getenv("LLM_PROVIDER")
+            main_model = os.getenv("LLM_MODEL")
+            if main_provider and main_model:
+                self.smart_model = f"{main_provider}/{main_model}"
+            
+        if not self.smart_model:
+            raise ValueError("Configuration Error: 'smart_model' not found. Set SMART_LLM_MODEL or LLM_PROVIDER/LLM_MODEL.")
+            
+        # Simple heuristic threshold (word count)
+        self.complexity_threshold = config.get("complexity_threshold", 20)
+
+    def select_model(self, query: str) -> str:
+        """
+        Selects a model based on query complexity.
+        This is a simple implementation of 'Dynamic Model Switching'.
+        """
+        # 1. Check length
+        word_count = len(query.split())
+        
+        if word_count < self.complexity_threshold:
+            print(f"   [Optimization] Routing to FAST model ({self.fast_model}) for simple query.")
+            return self.fast_model
+            
+        # 2. Check for complexity keywords
+        complex_terms = ["analyze", "reason", "plan", "code", "compare", "evaluate"]
+        if any(term in query.lower() for term in complex_terms):
+            print(f"   [Optimization] Routing to SMART model ({self.smart_model}) for reasoning task.")
+            return self.smart_model
+            
+        # Default to fast for everything else
+        print(f"   [Optimization] Defaulting to FAST model.")
+        return self.fast_model
+
+    async def route(self, query: str, framework) -> Dict[str, Any]:
+        """
+        Executes the query using the selected model.
+        """
+        selected_model = self.select_model(query)
+        
+        # In a real system, we would instantiate a temporary agent or use the LLM directly.
+        # Here we simulate the selection impacting the framework's execution.
+        
+        # TODO: This method signature might need adjustment to integrate deeply with Kite.
+        # For this demo, it returns the model name for the agent to use.
+        return {"model": selected_model}

kite/persistence.py

@@ -0,0 +1,42 @@
+"""
+Persistence utilities for saving and loading application state.
+Supports the "Pause & Resume" (Checkpointing) pattern.
+"""
+import json
+import os
+from typing import Any, Dict, Optional
+
+class JSONCheckpointer:
+    """
+    Simple file-based persistence for arbitrary state dictionaries.
+    """
+    def __init__(self, filepath: str):
+        self.filepath = filepath
+
+    def save(self, state: Dict[str, Any]) -> None:
+        """Save the state dictionary to JSON file."""
+        try:
+            with open(self.filepath, "w") as f:
+                json.dump(state, f, indent=2)
+            # print(f"      [Checkpoint] Saved to {self.filepath}")
+        except Exception as e:
+            print(f"      [Checkpoint Error] Failed to save: {e}")
+
+    def load(self) -> Optional[Dict[str, Any]]:
+        """Load the state dictionary from JSON file. Returns None if not found."""
+        if not os.path.exists(self.filepath):
+            return None
+            
+        try:
+            with open(self.filepath, "r") as f:
+                data = json.load(f)
+            print(f"      [Checkpoint] Resuming from {self.filepath}")
+            return data
+        except Exception as e:
+            print(f"      [Checkpoint Error] Corrupt file {self.filepath}: {e}")
+            return None
+
+    def clear(self) -> None:
+        """Remove the checkpoint file."""
+        if os.path.exists(self.filepath):
+            os.remove(self.filepath)

kite/pipeline/__init__.py

@@ -0,0 +1,5 @@
+"""Pipeline patterns module."""
+from .deterministic_pipeline import DeterministicPipeline
+from .reactive_pipeline import ReactivePipeline
+
+__all__ = ['DeterministicPipeline', 'ReactivePipeline']

kite/pipeline/deterministic_pipeline.py

@@ -0,0 +1,323 @@
+"""
+Deterministic Pipeline Pattern
+Level 1 Autonomy: The assembly line pattern with ZERO risk.
+
+Flow: Input -> Step 1 -> Step 2 -> ... -> Action
+- No loops
+- No choices
+- Precise, predictable execution
+"""
+
+import os
+import json
+from typing import Dict, List, Optional, Any, Callable
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+
+class PipelineStatus(Enum):
+    """Generic pipeline processing status."""
+    PENDING = "pending"
+    PROCESSING = "processing"
+    AWAITING_APPROVAL = "awaiting_approval"
+    SUSPENDED = "suspended"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    ERROR = "error"
+
+@dataclass
+class PipelineState:
+    """Current state of data in pipeline."""
+    task_id: str
+    status: PipelineStatus = PipelineStatus.PENDING
+    data: Any = None
+    results: Dict[str, Any] = field(default_factory=dict)
+    errors: List[str] = field(default_factory=list)
+    current_step_index: int = 0
+    feedback: Optional[str] = None
+    created_at: datetime = field(default_factory=datetime.now)
+    updated_at: datetime = field(default_factory=datetime.now)
+    
+    def __getitem__(self, key):
+        """Allow dict-like access for backward compatibility."""
+        return getattr(self, key)
+
+class DeterministicPipeline:
+    """
+    A generic deterministic processing pipeline.
+    
+    Level 1 Autonomy:
+    - Fixed sequence
+    - No loops
+    - Predictable
+    
+    Example:
+        pipeline = DeterministicPipeline("data_processor")
+        pipeline.add_step("load", load_func)
+        pipeline.add_step("process", process_func)
+        
+        result = pipeline.execute(raw_data)
+    """
+    
+    def __init__(self, name: str = "pipeline", event_bus = None):
+        self.name = name
+        self.event_bus = event_bus
+        self.steps: List[tuple[str, Callable]] = []
+        self.checkpoints: Dict[str, bool] = {}  # step_name -> approval_required
+        self.intervention_points: Dict[str, Callable] = {}  # step_name -> callback
+        self.history: List[PipelineState] = []
+        if self.event_bus:
+            self.event_bus.emit("pipeline:init", {"pipeline": self.name})
+    
+    def add_step(self, name: str, func: Callable):
+        """Add a step to the pipeline."""
+        self.steps.append((name, func))
+        print(f"  [OK] Added step: {name}")
+
+    def add_checkpoint(self, step_name: str, approval_required: bool = True):
+        """Pause execution for approval after this step."""
+        self.checkpoints[step_name] = approval_required
+        print(f"  [OK] Added checkpoint after: {step_name}")
+
+    def add_intervention_point(self, step_name: str, callback: Callable):
+        """Call a callback for user intervention before this step."""
+        self.intervention_points[step_name] = callback
+        print(f"  [OK] Added intervention point before: {step_name}")
+    
+    def execute(self, data: Any, task_id: Optional[str] = None) -> PipelineState:
+        """Execute all steps in the pipeline sequentially."""
+        t_id = task_id or f"TASK-{len(self.history)+1:04d}"
+        if self.event_bus:
+            self.event_bus.emit("pipeline:start", {"pipeline": self.name, "task_id": t_id, "data": str(data)[:100]})
+            self.event_bus.emit("pipeline:structure", {
+                "pipeline": self.name,
+                "task_id": t_id,
+                "steps": [name for name, _ in self.steps]
+            })
+        
+        state = PipelineState(task_id=t_id, data=data)
+        state.status = PipelineStatus.PROCESSING
+        self.history.append(state)
+        
+        return self._run_sync(state)
+
+    def resume(self, task_id: str, feedback: Optional[str] = None) -> PipelineState:
+        """Resume a suspended or awaiting_approval task (sync)."""
+        state = next((s for s in self.history if s.task_id == task_id), None)
+        if not state:
+            raise ValueError(f"Task ID {task_id} not found in history")
+        
+        if state.status not in [PipelineStatus.SUSPENDED, PipelineStatus.AWAITING_APPROVAL]:
+            print(f"[WARNING] Task {task_id} is in status {state.status}, not suspended.")
+            return state
+
+        print(f"\n[RESUME] Resuming pipeline: {self.name} (Task: {task_id})")
+        state.status = PipelineStatus.PROCESSING
+        state.feedback = feedback
+        
+        return self._run_sync(state)
+
+    def _run_sync(self, state: PipelineState) -> PipelineState:
+        """Internal runner for sync execution."""
+        try:
+            while state.current_step_index < len(self.steps):
+                step_idx = state.current_step_index
+                step_name, func = self.steps[step_idx]
+                
+                # 1. Intervention Point
+                if step_name in self.intervention_points:
+                    print(f"     [INTERVENTION] Triggering before: {step_name}...")
+                    callback = self.intervention_points[step_name]
+                    import inspect
+                    if inspect.iscoroutinefunction(callback):
+                        raise RuntimeError(f"Intervention callback for '{step_name}' is async. Use execute_async().")
+                    callback(state)
+
+                current_data = state.results[self.steps[step_idx-1][0]] if step_idx > 0 else state.data
+                
+                if self.event_bus:
+                    self.event_bus.emit("pipeline:step_start", {
+                        "pipeline": self.name,
+                        "task_id": state.task_id,
+                        "step": step_name,
+                        "index": step_idx
+                    })
+                    raise RuntimeError(f"Step '{step_name}' is async. Use execute_async().")
+                
+                result = func(current_data)
+                state.results[step_name] = result
+
+                if self.event_bus:
+                    self.event_bus.emit("pipeline:step", {
+                        "pipeline": self.name, 
+                        "task_id": state.task_id,
+                        "step": step_name,
+                        "result": str(result)[:200]
+                    })
+                state.current_step_index += 1
+                state.updated_at = datetime.now()
+
+                # 3. Checkpoint
+                if step_name in self.checkpoints:
+                    approval_req = self.checkpoints[step_name]
+                    if approval_req:
+                        if self.event_bus:
+                            self.event_bus.emit("pipeline:checkpoint", {"pipeline": self.name, "task_id": state.task_id, "step": step_name})
+                        state.status = PipelineStatus.AWAITING_APPROVAL
+                        return state
+                    else:
+                        state.status = PipelineStatus.SUSPENDED
+                        return state
+            
+            state.status = PipelineStatus.COMPLETED
+            if self.event_bus:
+                self.event_bus.emit("pipeline:complete", {"pipeline": self.name, "task_id": state.task_id})
+            
+        except Exception as e:
+            state.status = PipelineStatus.ERROR
+            state.errors.append(str(e))
+            state.updated_at = datetime.now()
+            print(f"[ERROR] Pipeline '{self.name}' failed: {e}")
+            
+        return state
+
+    async def execute_async(self, data: Any, task_id: Optional[str] = None) -> PipelineState:
+        """Execute all steps in the pipeline asynchronously."""
+        t_id = task_id or f"TASK-{len(self.history)+1:04d}"
+        if self.event_bus:
+            self.event_bus.emit("pipeline:start", {"pipeline": self.name, "task_id": t_id, "data": str(data)[:100], "mode": "async"})
+            self.event_bus.emit("pipeline:structure", {
+                "pipeline": self.name,
+                "task_id": t_id,
+                "steps": [name for name, _ in self.steps]
+            })
+        
+        state = PipelineState(task_id=t_id, data=data)
+        state.status = PipelineStatus.PROCESSING
+        self.history.append(state)
+        
+        return await self._run_async(state)
+
+    async def resume_async(self, task_id: str, feedback: Optional[str] = None) -> PipelineState:
+        """Resume a suspended or awaiting_approval task."""
+        state = next((s for s in self.history if s.task_id == task_id), None)
+        if not state:
+            raise ValueError(f"Task ID {task_id} not found in history")
+        
+        if state.status not in [PipelineStatus.SUSPENDED, PipelineStatus.AWAITING_APPROVAL]:
+            print(f"[WARNING] Task {task_id} is in status {state.status}, not suspended.")
+            return state
+
+        print(f"\n[RESUME] Resuming pipeline async: {self.name} (Task: {task_id})")
+        state.status = PipelineStatus.PROCESSING
+        state.feedback = feedback
+        
+        return await self._run_async(state)
+
+    async def _run_async(self, state: PipelineState) -> PipelineState:
+        """Internal runner for async execution."""
+        try:
+            while state.current_step_index < len(self.steps):
+                step_idx = state.current_step_index
+                step_name, func = self.steps[step_idx]
+                
+                # 1. Intervention Point (Before Step)
+                if step_name in self.intervention_points:
+                    print(f"     [INTERVENTION] Triggering before: {step_name}...")
+                    callback = self.intervention_points[step_name]
+                    # We pass the state and results for human to tweak
+                    await self._invoke_callback(callback, state)
+
+                # 2. Execute Step
+                current_data = state.results[self.steps[step_idx-1][0]] if step_idx > 0 else state.data
+                
+                # Create context for the step (useful for agents)
+                step_context = {
+                    "task_id": state.task_id,
+                    "pipeline": self.name,
+                    "step": step_name,
+                    "index": step_idx
+                }
+                
+                if self.event_bus:
+                    self.event_bus.emit("pipeline:step_start", {
+                        "pipeline": self.name,
+                        "task_id": state.task_id,
+                        "step": step_name,
+                        "index": step_idx
+                    })
+                
+                import inspect
+                # We need to decide how to pass context. Many steps in the scraper take only one arg.
+                # If it's an agent.run, we can pass context as a second arg.
+                # For now, let's just emit and assume the agent might pick it up if we change how we call it in the scraper,
+                # OR we can try to inject it if the function signature allows.
+                
+                if inspect.iscoroutinefunction(func):
+                    # Try to pass context if it's an agent-like run
+                    try:
+                        result = await func(current_data, context=step_context)
+                    except TypeError:
+                        result = await func(current_data)
+                else:
+                    try:
+                        result = func(current_data, context=step_context)
+                    except TypeError:
+                        result = func(current_data)
+                
+                state.results[step_name] = result
+
+                if self.event_bus:
+                    self.event_bus.emit("pipeline:step", {
+                        "pipeline": self.name, 
+                        "task_id": state.task_id,
+                        "step": step_name,
+                        "result": str(result)[:200]
+                    })
+                state.current_step_index += 1
+                state.updated_at = datetime.now()
+
+                # 3. Checkpoint (After Step)
+                if step_name in self.checkpoints:
+                    approval_req = self.checkpoints[step_name]
+                    if approval_req:
+                        if self.event_bus:
+                            self.event_bus.emit("pipeline:checkpoint", {"pipeline": self.name, "task_id": state.task_id, "step": step_name})
+                        state.status = PipelineStatus.AWAITING_APPROVAL
+                        return state
+                    else:
+                        state.status = PipelineStatus.SUSPENDED
+                        return state
+            
+            state.status = PipelineStatus.COMPLETED
+            if self.event_bus:
+                self.event_bus.emit("pipeline:complete", {"pipeline": self.name, "task_id": state.task_id})
+            
+        except Exception as e:
+            state.status = PipelineStatus.ERROR
+            state.errors.append(str(e))
+            state.updated_at = datetime.now()
+            print(f"[ERROR] Pipeline '{self.name}' failed: {e}")
+            
+        return state
+
+    async def _invoke_callback(self, callback: Callable, state: PipelineState):
+        """Invoke intervention callback."""
+        import inspect
+        if inspect.iscoroutinefunction(callback):
+            await callback(state)
+        else:
+            callback(state)
+    
+    def get_stats(self) -> Dict:
+        """Get pipeline statistics."""
+        if not self.history:
+            return {"total_processed": 0}
+        
+        statuses = [state.status for state in self.history]
+        return {
+            "total_processed": len(self.history),
+            "completed": statuses.count(PipelineStatus.COMPLETED),
+            "errors": statuses.count(PipelineStatus.ERROR),
+            "success_rate": (statuses.count(PipelineStatus.COMPLETED) / len(statuses) * 100) if statuses else 0
+        }