x-ipe 1.0.24__py3-none-any.whl → 1.0.25__py3-none-any.whl

x_ipe/services/kb_service.py

@@ -0,0 +1,378 @@
+"""
+FEATURE-025-A: KB Core Infrastructure
+
+KBService: Core Knowledge Base operations
+- Folder structure initialization
+- File index management  
+- Topic metadata management
+"""
+import json
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, List, Any, Optional
+
+from x_ipe.tracing import x_ipe_tracing
+
+
+class KBService:
+    """
+    Service for managing Knowledge Base infrastructure.
+    
+    Provides core operations for the x-ipe-docs/knowledge-base/ directory:
+    - initialize_structure(): Create KB folder structure
+    - get_index(): Get current file index
+    - refresh_index(): Rebuild index from file system
+    - get_topics(): List all topics
+    - get_topic_metadata(): Get topic metadata
+    """
+    
+    KB_PATH = 'x-ipe-docs/knowledge-base'
+    
+    # File type mapping
+    FILE_TYPE_MAP = {
+        # Documents
+        '.pdf': 'pdf',
+        '.md': 'markdown',
+        '.markdown': 'markdown',
+        '.txt': 'text',
+        '.docx': 'docx',
+        '.xlsx': 'xlsx',
+        
+        # Code
+        '.py': 'python',
+        '.js': 'javascript',
+        '.ts': 'typescript',
+        '.java': 'java',
+        '.go': 'go',
+        '.rs': 'rust',
+        '.c': 'c',
+        '.cpp': 'cpp',
+        '.h': 'header',
+        '.html': 'html',
+        '.css': 'css',
+        '.json': 'json',
+        '.yaml': 'yaml',
+        '.yml': 'yaml',
+        
+        # Images
+        '.png': 'image',
+        '.jpg': 'image',
+        '.jpeg': 'image',
+        '.gif': 'image',
+        '.svg': 'image',
+        '.webp': 'image',
+    }
+    
+    def __init__(self, project_root: str):
+        """
+        Initialize KBService.
+        
+        Args:
+            project_root: Absolute path to the project root directory
+        """
+        self.project_root = Path(project_root).resolve()
+        self.kb_root = self.project_root / self.KB_PATH
+        self.index_path = self.kb_root / 'index' / 'file-index.json'
+    
+    @x_ipe_tracing(level="INFO")
+    def initialize_structure(self) -> bool:
+        """
+        Create KB folder structure if it doesn't exist.
+        
+        Creates:
+        - landing/: Raw uploads
+        - topics/: Organized by topic
+        - processed/: AI summaries
+        - index/: Search index files
+        
+        Returns:
+            True if successful
+        """
+        try:
+            # Create main KB directory and subfolders
+            folders = [
+                self.kb_root / 'landing',
+                self.kb_root / 'topics',
+                self.kb_root / 'processed',
+                self.kb_root / 'index',
+            ]
+            
+            for folder in folders:
+                folder.mkdir(parents=True, exist_ok=True)
+            
+            # Create empty index if it doesn't exist
+            if not self.index_path.exists():
+                empty_index = {
+                    "version": "1.0",
+                    "last_updated": datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'),
+                    "files": []
+                }
+                self._write_json(self.index_path, empty_index)
+            
+            return True
+        except Exception as e:
+            # Log error but don't raise
+            return False
+    
+    @x_ipe_tracing(level="INFO")
+    def get_index(self) -> Dict:
+        """
+        Get current file index.
+        
+        If KB folder doesn't exist, initializes structure first.
+        If index is corrupted, recreates empty index.
+        
+        Returns:
+            File index dictionary with version, last_updated, and files list
+        """
+        # Initialize structure if needed
+        if not self.kb_root.exists():
+            self.initialize_structure()
+        
+        # Read index
+        try:
+            if self.index_path.exists():
+                with open(self.index_path, 'r', encoding='utf-8') as f:
+                    return json.load(f)
+        except (json.JSONDecodeError, IOError):
+            # Corrupted index - recreate
+            pass
+        
+        # Return/create empty index
+        empty_index = {
+            "version": "1.0",
+            "last_updated": datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'),
+            "files": []
+        }
+        self._write_json(self.index_path, empty_index)
+        return empty_index
+    
+    @x_ipe_tracing(level="INFO")
+    def refresh_index(self) -> Dict:
+        """
+        Rebuild index from file system.
+        
+        Scans landing/, topics/, and processed/ folders.
+        Updates topic metadata files.
+        
+        Returns:
+            Updated file index dictionary
+        """
+        # Initialize structure if needed
+        if not self.kb_root.exists():
+            self.initialize_structure()
+        
+        files = []
+        
+        # Scan landing folder
+        landing_path = self.kb_root / 'landing'
+        if landing_path.exists():
+            files.extend(self._scan_folder(landing_path, topic=None))
+        
+        # Scan topics folder
+        topics_path = self.kb_root / 'topics'
+        if topics_path.exists():
+            for topic_dir in topics_path.iterdir():
+                if topic_dir.is_dir() and not topic_dir.name.startswith('.'):
+                    topic_name = topic_dir.name
+                    # Scan raw subfolder
+                    raw_path = topic_dir / 'raw'
+                    if raw_path.exists():
+                        files.extend(self._scan_folder(raw_path, topic=topic_name))
+                    # Update topic metadata
+                    self._update_topic_metadata(topic_name)
+        
+        # Scan processed folder
+        processed_path = self.kb_root / 'processed'
+        if processed_path.exists():
+            for topic_dir in processed_path.iterdir():
+                if topic_dir.is_dir() and not topic_dir.name.startswith('.'):
+                    topic_name = topic_dir.name
+                    files.extend(self._scan_folder(topic_dir, topic=topic_name, prefix='processed'))
+        
+        # Build index
+        index = {
+            "version": "1.0",
+            "last_updated": datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'),
+            "files": files
+        }
+        
+        # Write index
+        self._write_json(self.index_path, index)
+        
+        return index
+    
+    @x_ipe_tracing(level="INFO")
+    def get_topics(self) -> List[str]:
+        """
+        Get list of all topic names.
+        
+        Returns:
+            List of topic folder names
+        """
+        topics = []
+        topics_path = self.kb_root / 'topics'
+        
+        if topics_path.exists():
+            for item in topics_path.iterdir():
+                if item.is_dir() and not item.name.startswith('.'):
+                    topics.append(item.name)
+        
+        return sorted(topics)
+    
+    @x_ipe_tracing(level="INFO")
+    def get_topic_metadata(self, topic: str) -> Optional[Dict]:
+        """
+        Get metadata for a specific topic.
+        
+        Args:
+            topic: Topic folder name
+            
+        Returns:
+            Topic metadata dictionary or None if not found
+        """
+        metadata_path = self.kb_root / 'topics' / topic / 'metadata.json'
+        
+        if not metadata_path.exists():
+            return None
+        
+        try:
+            with open(metadata_path, 'r', encoding='utf-8') as f:
+                return json.load(f)
+        except (json.JSONDecodeError, IOError):
+            return None
+    
+    def _scan_folder(self, folder: Path, topic: Optional[str], prefix: str = None) -> List[Dict]:
+        """
+        Scan a folder and return file entries.
+        
+        Args:
+            folder: Path to folder to scan
+            topic: Topic name (None for landing)
+            prefix: Path prefix (e.g., 'processed')
+            
+        Returns:
+            List of file entry dictionaries
+        """
+        files = []
+        
+        try:
+            for entry in folder.iterdir():
+                if entry.name.startswith('.'):
+                    continue  # Skip hidden files
+                
+                if entry.is_file():
+                    # Build relative path
+                    if prefix:
+                        rel_path = f"{prefix}/{topic}/{entry.name}"
+                    elif topic:
+                        rel_path = f"topics/{topic}/raw/{entry.name}"
+                    else:
+                        rel_path = f"landing/{entry.name}"
+                    
+                    # Get file stats
+                    stat = entry.stat()
+                    
+                    file_entry = {
+                        "path": rel_path,
+                        "name": entry.name,
+                        "type": self._get_file_type(entry.suffix),
+                        "size": stat.st_size,
+                        "topic": topic,
+                        "created_date": datetime.fromtimestamp(
+                            stat.st_mtime, tz=timezone.utc
+                        ).isoformat().replace('+00:00', 'Z'),
+                        "keywords": self._extract_keywords(entry.name)
+                    }
+                    files.append(file_entry)
+        except PermissionError:
+            pass  # Skip folders we can't read
+        
+        return files
+    
+    def _get_file_type(self, extension: str) -> str:
+        """
+        Get file type from extension.
+        
+        Args:
+            extension: File extension (e.g., '.pdf')
+            
+        Returns:
+            File type string (e.g., 'pdf', 'unknown')
+        """
+        return self.FILE_TYPE_MAP.get(extension.lower(), 'unknown')
+    
+    def _extract_keywords(self, filename: str) -> List[str]:
+        """
+        Extract keywords from filename.
+        
+        Splits on hyphens, underscores, spaces, and removes extension.
+        
+        Args:
+            filename: Filename to extract keywords from
+            
+        Returns:
+            List of lowercase keyword strings
+        """
+        # Remove extension
+        name_without_ext = Path(filename).stem
+        
+        # Split on common separators
+        parts = re.split(r'[-_\s]+', name_without_ext)
+        
+        # Filter and lowercase
+        keywords = [p.lower() for p in parts if p]
+        
+        return keywords
+    
+    def _update_topic_metadata(self, topic: str) -> None:
+        """
+        Update or create metadata.json for a topic.
+        
+        Args:
+            topic: Topic folder name
+        """
+        topic_path = self.kb_root / 'topics' / topic
+        metadata_path = topic_path / 'metadata.json'
+        raw_path = topic_path / 'raw'
+        
+        # Count files in raw folder
+        file_count = 0
+        if raw_path.exists():
+            file_count = len([f for f in raw_path.iterdir() 
+                            if f.is_file() and not f.name.startswith('.')])
+        
+        # Load existing or create new
+        if metadata_path.exists():
+            try:
+                with open(metadata_path, 'r', encoding='utf-8') as f:
+                    metadata = json.load(f)
+            except (json.JSONDecodeError, IOError):
+                metadata = {}
+        else:
+            metadata = {}
+        
+        # Update fields
+        metadata.update({
+            "name": topic,
+            "description": metadata.get("description", ""),
+            "file_count": file_count,
+            "last_updated": datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z'),
+            "tags": metadata.get("tags", [])
+        })
+        
+        # Write
+        self._write_json(metadata_path, metadata)
+    
+    def _write_json(self, path: Path, data: Dict) -> None:
+        """
+        Write JSON data to file with pretty formatting.
+        
+        Args:
+            path: Path to write to
+            data: Dictionary to write
+        """
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with open(path, 'w', encoding='utf-8') as f:
+            json.dump(data, f, indent=2, ensure_ascii=False)

x_ipe/services/proxy_service.py

@@ -12,6 +12,8 @@ from bs4 import BeautifulSoup
 from dataclasses import dataclass
 from typing import Tuple
 
+from x_ipe.tracing import x_ipe_tracing
+
 ALLOWED_HOSTS = {'localhost', '127.0.0.1'}
 ALLOWED_SCHEMES = {'http', 'https', 'file'}
 PROXY_TIMEOUT = 10  # seconds
@@ -142,6 +144,7 @@ def _is_binary_content_type(content_type: str) -> bool:
 class ProxyService:
     """Service for proxying localhost URLs."""
     
+    @x_ipe_tracing()
     def validate_url(self, url: str) -> Tuple[bool, str]:
         """
         Validate URL is localhost or local file.
@@ -182,6 +185,7 @@ class ProxyService:
         except Exception as e:
             return False, f"Invalid URL format: {str(e)}"
     
+    @x_ipe_tracing()
     def fetch_and_rewrite(self, url: str) -> ProxyResult:
         """
         Fetch URL and rewrite asset paths for proxy.

x_ipe/services/settings_service.py

@@ -8,6 +8,8 @@ import os
 import sqlite3
 from typing import Dict, List, Optional, Any
 
+from x_ipe.tracing import x_ipe_tracing
+
 
 class SettingsService:
     """
@@ -74,6 +76,7 @@ class SettingsService:
         finally:
             conn.close()
     
+    @x_ipe_tracing()
     def get(self, key: str, default: Any = None) -> Optional[str]:
         """
         Get a single setting value.
@@ -94,6 +97,7 @@ class SettingsService:
         finally:
             conn.close()
     
+    @x_ipe_tracing()
     def get_all(self) -> Dict[str, str]:
         """
         Get all settings as dictionary.
@@ -109,6 +113,7 @@ class SettingsService:
         finally:
             conn.close()
     
+    @x_ipe_tracing()
     def set(self, key: str, value: str) -> None:
         """
         Set a single setting value.
@@ -133,6 +138,7 @@ class SettingsService:
         finally:
             conn.close()
     
+    @x_ipe_tracing()
     def validate_project_root(self, path: str) -> Dict[str, str]:
         """
         Validate project root path.
@@ -265,6 +271,7 @@ class ProjectFoldersService:
         finally:
             conn.close()
     
+    @x_ipe_tracing()
     def get_all(self) -> List[Dict]:
         """
         Get all project folders.
@@ -280,6 +287,7 @@ class ProjectFoldersService:
         finally:
             conn.close()
     
+    @x_ipe_tracing()
     def get_by_id(self, project_id: int) -> Optional[Dict]:
         """
         Get project folder by ID.
@@ -299,6 +307,7 @@ class ProjectFoldersService:
         finally:
             conn.close()
     
+    @x_ipe_tracing()
     def add(self, name: str, path: str) -> Dict:
         """
         Add a new project folder.
@@ -333,6 +342,7 @@ class ProjectFoldersService:
         finally:
             conn.close()
     
+    @x_ipe_tracing()
     def update(self, project_id: int, name: str = None, path: str = None) -> Dict:
         """
         Update an existing project folder.
@@ -374,6 +384,7 @@ class ProjectFoldersService:
         finally:
             conn.close()
     
+    @x_ipe_tracing()
     def delete(self, project_id: int, active_project_id: int = None) -> Dict:
         """
         Delete a project folder.
@@ -437,6 +448,7 @@ class ProjectFoldersService:
         
         return errors
     
+    @x_ipe_tracing()
     def get_active_id(self) -> int:
         """
         Get the active project ID from settings.
@@ -453,6 +465,7 @@ class ProjectFoldersService:
         finally:
             conn.close()
     
+    @x_ipe_tracing()
     def set_active(self, project_id: int) -> Dict:
         """
         Set the active project ID.

x_ipe/services/skills_service.py

@@ -5,6 +5,8 @@ import yaml
 from pathlib import Path
 from typing import Dict, List, Optional
 
+from x_ipe.tracing import x_ipe_tracing
+
 
 class SkillsService:
     """
@@ -16,6 +18,7 @@ class SkillsService:
     
     SKILLS_PATH = '.github/skills'
     
+    @x_ipe_tracing()
     def __init__(self, project_root: str):
         """
         Initialize SkillsService.
@@ -26,6 +29,7 @@ class SkillsService:
         self.project_root = Path(project_root).resolve()
         self.skills_dir = self.project_root / self.SKILLS_PATH
     
+    @x_ipe_tracing()
     def get_all(self) -> List[Dict[str, str]]:
         """
         Get all skills with their name and description.

x_ipe/services/terminal_service.py

@@ -13,6 +13,8 @@ from collections import deque
 from datetime import datetime
 from typing import Dict, Optional, Any, Callable
 
+from x_ipe.tracing import x_ipe_tracing
+
 
 # Constants for session management
 BUFFER_MAX_CHARS = 10240  # 10KB limit for output buffer
@@ -30,15 +32,18 @@ class OutputBuffer:
     def __init__(self, max_chars: int = BUFFER_MAX_CHARS):
         self._buffer: deque = deque(maxlen=max_chars)
     
+    @x_ipe_tracing()
     def append(self, data: str) -> None:
         """Append data character by character to maintain limit."""
         for char in data:
             self._buffer.append(char)
     
+    @x_ipe_tracing()
     def get_contents(self) -> str:
         """Get all buffered content as string."""
         return ''.join(self._buffer)
     
+    @x_ipe_tracing()
     def clear(self) -> None:
         """Clear the buffer."""
         self._buffer.clear()
@@ -68,6 +73,7 @@ class PersistentSession:
         self.created_at = datetime.now()
         self._lock = threading.Lock()
     
+    @x_ipe_tracing()
     def start_pty(self, rows: int = 24, cols: int = 80) -> None:
         """Start the underlying PTY process."""
         def buffered_emit(data: str) -> None:
@@ -80,6 +86,7 @@ class PersistentSession:
         self.pty_session = PTYSession(self.session_id, buffered_emit)
         self.pty_session.start(rows, cols)
     
+    @x_ipe_tracing()
     def attach(self, socket_sid: str, emit_callback: Callable[[str], None]) -> None:
         """Attach a WebSocket connection to this session."""
         with self._lock:
@@ -88,6 +95,7 @@ class PersistentSession:
             self.state = 'connected'
             self.disconnect_time = None
     
+    @x_ipe_tracing()
     def detach(self) -> None:
         """Detach WebSocket, keeping PTY alive for reconnection."""
         with self._lock:
@@ -96,20 +104,24 @@ class PersistentSession:
             self.state = 'disconnected'
             self.disconnect_time = datetime.now()
     
+    @x_ipe_tracing()
     def get_buffer(self) -> str:
         """Get buffered output for replay."""
         return self.output_buffer.get_contents()
     
+    @x_ipe_tracing()
     def write(self, data: str) -> None:
         """Write input to PTY."""
         if self.pty_session:
             self.pty_session.write(data)
     
+    @x_ipe_tracing()
     def resize(self, rows: int, cols: int) -> None:
         """Resize the PTY."""
         if self.pty_session:
             self.pty_session._set_size(rows, cols)
     
+    @x_ipe_tracing()
     def is_expired(self, timeout_seconds: int = SESSION_TIMEOUT) -> bool:
         """Check if session has expired (1hr after disconnect)."""
         if self.state == 'connected':
@@ -119,6 +131,7 @@ class PersistentSession:
         elapsed = datetime.now() - self.disconnect_time
         return elapsed.total_seconds() > timeout_seconds
     
+    @x_ipe_tracing()
     def close(self) -> None:
         """Close session and cleanup resources."""
         if self.pty_session:
@@ -139,6 +152,7 @@ class SessionManager:
         self._cleanup_timer: Optional[threading.Timer] = None
         self._running = False
     
+    @x_ipe_tracing()
     def create_session(self, emit_callback: Callable[[str], None],
                        rows: int = 24, cols: int = 80) -> str:
         """Create new persistent session, returns session_id."""
@@ -152,16 +166,19 @@ class SessionManager:
         
         return session_id
     
+    @x_ipe_tracing()
     def get_session(self, session_id: str) -> Optional[PersistentSession]:
         """Get session by ID."""
         with self._lock:
             return self.sessions.get(session_id)
     
+    @x_ipe_tracing()
     def has_session(self, session_id: str) -> bool:
         """Check if session exists."""
         with self._lock:
             return session_id in self.sessions
     
+    @x_ipe_tracing()
     def remove_session(self, session_id: str) -> None:
         """Remove and close a session."""
         with self._lock:
@@ -169,6 +186,7 @@ class SessionManager:
         if session:
             session.close()
     
+    @x_ipe_tracing()
     def cleanup_expired(self) -> int:
         """Remove expired sessions. Returns count removed."""
         expired_ids = []
@@ -182,11 +200,13 @@ class SessionManager:
         
         return len(expired_ids)
     
+    @x_ipe_tracing()
     def start_cleanup_task(self) -> None:
         """Start background cleanup task (every 5 minutes)."""
         self._running = True
         self._schedule_cleanup()
     
+    @x_ipe_tracing()
     def stop_cleanup_task(self) -> None:
         """Stop the cleanup task."""
         self._running = False
@@ -230,6 +250,7 @@ class PTYSession:
         self.rows = 24
         self.cols = 80
     
+    @x_ipe_tracing()
     def start(self, rows: int = 24, cols: int = 80) -> None:
         """Spawn PTY with shell and start output reader."""
         import pty
@@ -310,6 +331,7 @@ class PTYSession:
         except Exception:
             pass
     
+    @x_ipe_tracing()
     def write(self, data: str) -> None:
         """Write input to PTY."""
         if self.fd is not None:
@@ -328,6 +350,7 @@ class PTYSession:
             winsize = struct.pack('HHHH', self.rows, self.cols, 0, 0)
             fcntl.ioctl(self.fd, termios.TIOCSWINSZ, winsize)
     
+    @x_ipe_tracing()
     def close(self) -> None:
         """Terminate PTY session and cleanup."""
         import signal
@@ -351,6 +374,7 @@ class PTYSession:
                 pass
             self.pid = None
     
+    @x_ipe_tracing()
     def isalive(self) -> bool:
         """Check if the PTY process is still running."""
         return self._running and self.fd is not None

x_ipe/services/themes_service.py

@@ -10,6 +10,8 @@ import re
 from pathlib import Path
 from typing import Dict, List, Any, Optional
 
+from x_ipe.tracing import x_ipe_tracing
+
 
 THEMES_DIR = 'x-ipe-docs/themes'
 THEME_PREFIX = 'theme-'
@@ -45,6 +47,7 @@ class ThemesService:
         self.project_root = Path(project_root).resolve()
         self.themes_dir = self.project_root / THEMES_DIR
     
+    @x_ipe_tracing()
     def list_themes(self) -> List[Dict[str, Any]]:
         """
         List all valid themes with metadata.
@@ -92,6 +95,7 @@ class ThemesService:
         
         return themes
     
+    @x_ipe_tracing()
     def get_theme(self, name: str) -> Optional[Dict[str, Any]]:
         """
         Get detailed information about a specific theme.