x-ipe 1.0.24__py3-none-any.whl → 1.0.25__py3-none-any.whl

x_ipe/tracing/middleware.py

@@ -0,0 +1,197 @@
+"""
+FEATURE-023: Application Action Tracing - Middleware
+
+Flask middleware for automatic request tracing.
+
+Creates TraceContext for each request when tracing is active,
+then writes the trace log file after the request completes.
+"""
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from flask import Flask, request, g
+from typing import List
+
+from .context import TraceContext
+from .writer import TraceLogWriter
+
+
+# APIs to ignore (to avoid infinite loops and noise)
+IGNORED_API_PREFIXES = (
+    '/api/tracing/',      # Tracing APIs themselves
+    '/static/',           # Static files
+    '/socket.io/',        # WebSocket
+)
+
+
+def init_tracing_middleware(app: Flask) -> None:
+    """
+    Initialize tracing middleware for a Flask app.
+    
+    Registers before_request and after_request hooks that:
+    - Check if tracing is active
+    - Start TraceContext before each request
+    - End TraceContext and write log file after each request
+    
+    Args:
+        app: Flask application instance
+    """
+    
+    @app.before_request
+    def start_trace():
+        """Start trace context if tracing is active."""
+        # Skip hardcoded ignored paths (system paths)
+        if request.path.startswith(IGNORED_API_PREFIXES):
+            return
+        
+        # Check if tracing is active and get config
+        project_root = app.config.get('PROJECT_ROOT', '.')
+        active, ignored_apis = _get_tracing_config(project_root)
+        
+        if not active:
+            return
+        
+        # Skip user-configured ignored APIs
+        if _is_path_ignored(request.path, ignored_apis):
+            return
+        
+        # Start trace context
+        api_name = f"{request.method} {request.path}"
+        TraceContext.start_trace(api_name)
+        g.trace_start_time = time.perf_counter()
+    
+    @app.after_request
+    def end_trace(response):
+        """End trace context and write log file."""
+        # Check if we started a trace
+        if not hasattr(g, 'trace_start_time'):
+            return response
+        
+        # End the trace
+        buffer = TraceContext.end_trace()
+        if buffer is None:
+            return response
+        
+        # Determine status from response
+        status = "SUCCESS" if response.status_code < 400 else "ERROR"
+        
+        # Get log path from config
+        project_root = app.config.get('PROJECT_ROOT', '.')
+        log_path = _get_trace_log_path(project_root)
+        
+        # Write trace to file
+        writer = TraceLogWriter(log_path)
+        writer.write(buffer, status)
+        
+        return response
+    
+    @app.teardown_request
+    def cleanup_trace(exception=None):
+        """Clean up trace context on error."""
+        if exception is not None:
+            buffer = TraceContext.end_trace()
+            if buffer is not None:
+                project_root = app.config.get('PROJECT_ROOT', '.')
+                log_path = _get_trace_log_path(project_root)
+                writer = TraceLogWriter(log_path)
+                writer.write(buffer, "ERROR")
+
+
+def _get_tracing_config(project_root: str) -> tuple:
+    """
+    Get tracing configuration including active status and ignored APIs.
+    
+    Reads from tools.json to check tracing_enabled, tracing_stop_at,
+    and tracing_ignored_apis.
+    
+    Args:
+        project_root: Path to project root
+        
+    Returns:
+        Tuple of (is_active: bool, ignored_apis: List[str])
+    """
+    import json
+    
+    tools_path = Path(project_root) / "x-ipe-docs" / "config" / "tools.json"
+    if not tools_path.exists():
+        return False, []
+    
+    try:
+        with open(tools_path) as f:
+            config = json.load(f)
+    except (json.JSONDecodeError, IOError):
+        return False, []
+    
+    ignored_apis = config.get("tracing_ignored_apis", [])
+    
+    # Check if explicitly enabled
+    if config.get("tracing_enabled", False):
+        return True, ignored_apis
+    
+    # Check if stop_at is in the future
+    stop_at = config.get("tracing_stop_at")
+    if stop_at:
+        try:
+            # Keep timezone-aware for proper comparison
+            stop_time = datetime.fromisoformat(
+                stop_at.replace("Z", "+00:00")
+            )
+            if datetime.now(timezone.utc) < stop_time:
+                return True, ignored_apis
+        except (ValueError, AttributeError):
+            pass
+    
+    return False, ignored_apis
+
+
+def _is_path_ignored(path: str, ignored_apis: List[str]) -> bool:
+    """
+    Check if a request path matches any user-configured ignored API patterns.
+    
+    Supports exact matches and prefix matches (patterns ending with *).
+    
+    Args:
+        path: Request path (e.g., "/api/project/structure")
+        ignored_apis: List of API patterns to ignore
+        
+    Returns:
+        True if path should be ignored
+    """
+    for pattern in ignored_apis:
+        if pattern.endswith('*'):
+            # Prefix match
+            if path.startswith(pattern[:-1]):
+                return True
+        else:
+            # Exact match
+            if path == pattern:
+                return True
+    return False
+
+
+def _get_trace_log_path(project_root: str) -> str:
+    """
+    Get the trace log directory path.
+    
+    Reads from tools.json or uses default.
+    
+    Args:
+        project_root: Path to project root
+        
+    Returns:
+        Absolute path to trace log directory
+    """
+    import json
+    
+    tools_path = Path(project_root) / "x-ipe-docs" / "config" / "tools.json"
+    log_path = "instance/traces/"
+    
+    if tools_path.exists():
+        try:
+            with open(tools_path) as f:
+                config = json.load(f)
+                log_path = config.get("tracing_log_path", log_path)
+        except (json.JSONDecodeError, IOError):
+            pass
+    
+    return str(Path(project_root) / log_path)

x_ipe/tracing/parser.py

@@ -0,0 +1,235 @@
+"""
+FEATURE-023-C: Trace Viewer & DAG Visualization
+
+TraceLogParser for parsing trace log files into visualization-ready graph structures.
+
+Handles the log format:
+    [TRACE-START] trace_id | API | timestamp
+    [INFO] → start_function: name | input_json
+    [INFO] ← return_function: name | output_json | duration
+    [ERROR] ← exception: name | error | duration
+    [TRACE-END] trace_id | total_duration | status
+"""
+import re
+from pathlib import Path
+from typing import Dict, List, Any, Optional
+
+
+class TraceLogParser:
+    """
+    Parse trace log files into graph structure for visualization.
+    
+    Usage:
+        parser = TraceLogParser()
+        result = parser.parse(Path("trace.log"))
+        # result = {"trace_id": "...", "nodes": [...], "edges": [...]}
+    """
+    
+    # Regex patterns for parsing log lines
+    TRACE_START_PATTERN = re.compile(
+        r'\[TRACE-START\]\s*([^\|]+)\s*\|\s*([^\|]+)\s*\|\s*(.+)'
+    )
+    TRACE_END_PATTERN = re.compile(
+        r'\[TRACE-END\]\s*([^\|]+)\s*\|\s*(\d+)ms\s*\|\s*(\w+)'
+    )
+    FUNCTION_START_PATTERN = re.compile(
+        r'\[(INFO|DEBUG)\]\s*→\s*start_function:\s*([^\|]+)\s*\|\s*(.+)'
+    )
+    FUNCTION_RETURN_PATTERN = re.compile(
+        r'\[(INFO|DEBUG)\]\s*←\s*return_function:\s*([^\|]+)\s*\|\s*([^\|]+)\s*\|\s*(\d+)ms'
+    )
+    EXCEPTION_PATTERN = re.compile(
+        r'\[(ERROR|INFO|DEBUG)\]\s*←\s*exception:\s*([^\|]+)\s*\|\s*([^|]+):\s*([^\|]+)\s*\|\s*(\d+)ms'
+    )
+    STACK_LINE_PATTERN = re.compile(
+        r'\s+at\s+(\w+)\s+\(([^:]+):?(\d+)?\)'
+    )
+    
+    def parse(self, filepath: Path) -> Dict[str, Any]:
+        """
+        Parse trace log file into visualization-ready structure.
+        
+        Args:
+            filepath: Path to the trace log file
+            
+        Returns:
+            Dictionary with trace data:
+            {
+                "trace_id": str,
+                "api": str,
+                "timestamp": str,
+                "total_time_ms": int,
+                "status": str ("success" or "error"),
+                "nodes": [
+                    {
+                        "id": str,
+                        "label": str,
+                        "timing": str,
+                        "status": str,
+                        "level": str,
+                        "input": str,
+                        "output": str,
+                        "error": dict or None
+                    }
+                ],
+                "edges": [
+                    {"source": str, "target": str}
+                ]
+            }
+        """
+        result = {
+            "trace_id": "",
+            "api": "",
+            "timestamp": "",
+            "total_time_ms": 0,
+            "status": "success",
+            "nodes": [],
+            "edges": []
+        }
+        
+        if not filepath.exists():
+            return result
+        
+        content = filepath.read_text()
+        if not content.strip():
+            return result
+        
+        lines = content.splitlines()
+        nodes = []
+        edges = []
+        call_stack = []  # Stack of node indices for tracking parent-child
+        node_counter = 0
+        current_error_node = None
+        stack_lines = []
+        
+        for raw_line in lines:
+            # Keep original line for stack trace matching
+            line = raw_line.strip()
+            
+            # Check for stack trace lines (indented with 'at')
+            stack_match = self.STACK_LINE_PATTERN.match(raw_line)
+            if stack_match and current_error_node is not None:
+                func, file_path, line_num = stack_match.groups()
+                stack_lines.append({
+                    "func": func,
+                    "file": file_path,
+                    "line": int(line_num) if line_num else None
+                })
+                continue
+            
+            # If we were collecting stack and hit non-stack line, finalize
+            if stack_lines and current_error_node is not None:
+                nodes[current_error_node]["error"]["stack"] = stack_lines
+                stack_lines = []
+                current_error_node = None
+            
+            # TRACE-START
+            start_match = self.TRACE_START_PATTERN.match(line)
+            if start_match:
+                trace_id, api, timestamp = start_match.groups()
+                result["trace_id"] = trace_id.strip()
+                result["api"] = api.strip()
+                result["timestamp"] = timestamp.strip()
+                
+                # Create root API node
+                nodes.append({
+                    "id": f"node-{node_counter}",
+                    "label": api.strip(),
+                    "timing": "",
+                    "status": "success",
+                    "level": "API",
+                    "input": "{}",
+                    "output": "{}",
+                    "error": None
+                })
+                call_stack.append(node_counter)
+                node_counter += 1
+                continue
+            
+            # TRACE-END
+            end_match = self.TRACE_END_PATTERN.match(line)
+            if end_match:
+                _, total_ms, status = end_match.groups()
+                result["total_time_ms"] = int(total_ms)
+                result["status"] = status.lower()
+                
+                # Update root node timing
+                if nodes:
+                    nodes[0]["timing"] = f"{total_ms}ms"
+                    if status.upper() == "ERROR":
+                        nodes[0]["status"] = "error"
+                continue
+            
+            # Function start
+            func_start_match = self.FUNCTION_START_PATTERN.match(line)
+            if func_start_match:
+                level, func_name, input_json = func_start_match.groups()
+                func_name = func_name.strip()
+                
+                # Create function node
+                new_node = {
+                    "id": f"node-{node_counter}",
+                    "label": func_name,
+                    "timing": "",
+                    "status": "success",
+                    "level": level,
+                    "input": input_json.strip(),
+                    "output": "{}",
+                    "error": None
+                }
+                nodes.append(new_node)
+                
+                # Create edge from parent
+                if call_stack:
+                    parent_id = call_stack[-1]
+                    edges.append({
+                        "source": f"node-{parent_id}",
+                        "target": f"node-{node_counter}"
+                    })
+                
+                call_stack.append(node_counter)
+                node_counter += 1
+                continue
+            
+            # Function return
+            func_return_match = self.FUNCTION_RETURN_PATTERN.match(line)
+            if func_return_match:
+                level, func_name, output_json, duration = func_return_match.groups()
+                func_name = func_name.strip()
+                
+                # Pop from stack and update node
+                if call_stack:
+                    current_id = call_stack.pop()
+                    if current_id < len(nodes):
+                        nodes[current_id]["output"] = output_json.strip()
+                        nodes[current_id]["timing"] = f"{duration}ms"
+                continue
+            
+            # Exception
+            exception_match = self.EXCEPTION_PATTERN.match(line)
+            if exception_match:
+                level, func_name, error_type, error_msg, duration = exception_match.groups()
+                func_name = func_name.strip()
+                
+                # Pop from stack and update node with error
+                if call_stack:
+                    current_id = call_stack.pop()
+                    if current_id < len(nodes):
+                        nodes[current_id]["status"] = "error"
+                        nodes[current_id]["timing"] = f"{duration}ms"
+                        nodes[current_id]["error"] = {
+                            "type": error_type.strip(),
+                            "message": error_msg.strip(),
+                            "stack": []
+                        }
+                        current_error_node = current_id
+                continue
+        
+        # Finalize any remaining stack lines
+        if stack_lines and current_error_node is not None:
+            nodes[current_error_node]["error"]["stack"] = stack_lines
+        
+        result["nodes"] = nodes
+        result["edges"] = edges
+        
+        return result

x_ipe/tracing/redactor.py

@@ -0,0 +1,111 @@
+"""
+FEATURE-023: Application Action Tracing - Core
+
+Redactor module for sensitive data redaction in trace logs.
+
+This module provides automatic redaction of sensitive data before logging,
+including passwords, tokens, API keys, and credit card numbers.
+"""
+import re
+from typing import Any, Dict, List, Set, Optional
+
+REDACTED = "[REDACTED]"
+
+# Sensitive key patterns (case-insensitive matching)
+SENSITIVE_KEY_PATTERNS = {
+    "password",
+    "secret",
+    "token",
+    "api_key",
+    "apikey",
+    "authorization",
+    "auth",
+    "credential",
+    "private_key",
+    "privatekey",
+}
+
+# Value patterns
+CREDIT_CARD_PATTERN = re.compile(r"^\d{16}$")
+JWT_PREFIX = "eyJ"
+
+
+class Redactor:
+    """
+    Sensitive data redactor for trace logs.
+    
+    Automatically redacts:
+    - Fields containing sensitive key patterns (password, secret, token, etc.)
+    - Credit card numbers (16-digit patterns)
+    - JWT tokens (values starting with 'eyJ')
+    - Custom fields specified via constructor
+    
+    Usage:
+        redactor = Redactor(custom_fields=["ssn", "dob"])
+        safe_data = redactor.redact({"password": "secret", "name": "John"})
+        # Result: {"password": "[REDACTED]", "name": "John"}
+    """
+    
+    def __init__(self, custom_fields: Optional[List[str]] = None):
+        """
+        Initialize Redactor with optional custom fields.
+        
+        Args:
+            custom_fields: Additional field names to redact (case-insensitive)
+        """
+        self.custom_fields: Set[str] = set(
+            f.lower() for f in (custom_fields or [])
+        )
+    
+    def redact(self, data: Any) -> Any:
+        """
+        Recursively redact sensitive data.
+        
+        Args:
+            data: Data to redact (dict, list, or primitive)
+            
+        Returns:
+            Data with sensitive values replaced with '[REDACTED]'
+        """
+        if isinstance(data, dict):
+            return {k: self._redact_value(k, v) for k, v in data.items()}
+        elif isinstance(data, list):
+            return [self.redact(item) for item in data]
+        elif isinstance(data, tuple):
+            return tuple(self.redact(item) for item in data)
+        return data
+    
+    def _redact_value(self, key: str, value: Any) -> Any:
+        """
+        Redact a single value based on key name and value patterns.
+        
+        Args:
+            key: Field name
+            value: Field value
+            
+        Returns:
+            Redacted value or original value if not sensitive
+        """
+        key_lower = key.lower()
+        
+        # Check field name against sensitive patterns
+        for pattern in SENSITIVE_KEY_PATTERNS:
+            if pattern in key_lower:
+                return REDACTED
+        
+        # Check custom fields
+        if key_lower in self.custom_fields:
+            return REDACTED
+        
+        # Check value patterns for strings
+        if isinstance(value, str):
+            # Credit card pattern (16 digits)
+            if CREDIT_CARD_PATTERN.match(value):
+                return REDACTED
+            
+            # JWT pattern (starts with eyJ)
+            if value.startswith(JWT_PREFIX):
+                return REDACTED
+        
+        # Recurse for nested structures
+        return self.redact(value)

x_ipe/tracing/writer.py

@@ -0,0 +1,122 @@
+"""
+FEATURE-023: Application Action Tracing - Core
+
+TraceLogWriter for writing trace buffers to log files.
+
+Handles file naming, directory creation, permission setting,
+and cleanup of old log files.
+"""
+import os
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from .buffer import TraceBuffer
+
+
+class TraceLogWriter:
+    """
+    Writer for trace log files.
+    
+    Writes trace buffers to structured log files with proper naming,
+    permissions, and cleanup of old files.
+    
+    Usage:
+        writer = TraceLogWriter("instance/traces/")
+        filepath = writer.write(buffer, "SUCCESS")
+        deleted = writer.cleanup(retention_hours=24)
+    """
+    
+    def __init__(self, log_path: str = "instance/traces/"):
+        """
+        Initialize TraceLogWriter.
+        
+        Args:
+            log_path: Directory path for log files
+        """
+        self.log_path = Path(log_path)
+    
+    def write(self, buffer: TraceBuffer, status: str = "SUCCESS") -> Optional[str]:
+        """
+        Write trace buffer to log file.
+        
+        Creates the log directory if it doesn't exist.
+        Sets file permissions to 600 (owner read/write only).
+        
+        Args:
+            buffer: TraceBuffer to write
+            status: Final status (SUCCESS, ERROR)
+            
+        Returns:
+            Path to created log file, or None if write failed
+        """
+        try:
+            # Ensure directory exists
+            self.log_path.mkdir(parents=True, exist_ok=True)
+            
+            # Calculate total duration
+            total_ms = (datetime.now(timezone.utc) - buffer.started_at).total_seconds() * 1000
+            content = buffer.to_log_string(status, total_ms)
+            
+            # Generate filename
+            timestamp = buffer.started_at.strftime("%Y%m%d-%H%M%S")
+            api_name = self._sanitize_api_name(buffer.root_api)
+            filename = f"{timestamp}-{api_name}-{buffer.trace_id}.log"
+            filepath = self.log_path / filename
+            
+            # Write file
+            with open(filepath, 'w', encoding='utf-8') as f:
+                f.write(content)
+            
+            # Set permissions (owner read/write only)
+            os.chmod(filepath, 0o600)
+            
+            return str(filepath)
+            
+        except Exception as e:
+            print(f"[TRACING] Failed to write log: {e}")
+            return None
+    
+    def cleanup(self, retention_hours: int = 24) -> int:
+        """
+        Delete log files older than retention period.
+        
+        Args:
+            retention_hours: Hours to retain log files
+            
+        Returns:
+            Number of files deleted
+        """
+        if not self.log_path.exists():
+            return 0
+        
+        deleted = 0
+        cutoff = datetime.now(timezone.utc).timestamp() - (retention_hours * 3600)
+        
+        for filepath in self.log_path.glob("*.log"):
+            try:
+                if filepath.stat().st_mtime < cutoff:
+                    filepath.unlink()
+                    deleted += 1
+            except OSError:
+                continue
+        
+        return deleted
+    
+    def _sanitize_api_name(self, api: str) -> str:
+        """
+        Convert API string to safe filename component.
+        
+        Args:
+            api: API string (e.g., "POST /api/orders")
+            
+        Returns:
+            Sanitized string (e.g., "post--api-orders")
+        """
+        # Replace spaces and slashes with hyphens
+        sanitized = api.lower().replace(" ", "-").replace("/", "-")
+        # Remove multiple consecutive hyphens
+        sanitized = re.sub(r'-+', '-', sanitized)
+        # Strip leading/trailing hyphens
+        return sanitized.strip("-")

{x_ipe-1.0.24.dist-info → x_ipe-1.0.25.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: x-ipe
-Version: 1.0.24
-Summary: X-IPE: AI-powered development framework
+Version: 1.0.25
+Summary: An AI native integrated project environment for end to end business value delivery
 License-File: LICENSE
 Requires-Python: >=3.12
 Requires-Dist: beautifulsoup4>=4.14.3