sql-xel-parser 1.0.0__py3-none-any.whl

sql_xel_parser/parser.py

@@ -0,0 +1,379 @@
+"""
+XEL Parser - Core module for parsing SQL Server Extended Events (.xel) files.
+
+This module handles the low-level parsing of XEL binary files and extraction
+of event data.
+"""
+
+import struct
+import sys
+import xml.etree.ElementTree as ET
+from typing import Iterator, Dict, Any, Optional, BinaryIO
+from datetime import datetime
+from io import BytesIO
+
+
+class XELParser:
+    """Parser for SQL Server Extended Events (.xel) files."""
+
+    # XEL file format constants
+    HEADER_MAGIC = b'XELH'
+    EVENT_MAGIC = b'XEVT'
+
+    def __init__(self, file_path: str):
+        """
+        Initialize the XEL parser.
+
+        Args:
+            file_path: Path to the .xel file
+        """
+        self.file_path = file_path
+        self.events_parsed = 0
+
+    def parse(self) -> Iterator[Dict[str, Any]]:
+        """
+        Parse the XEL file and yield events as dictionaries.
+
+        Yields:
+            Dictionary containing event data
+        """
+        events_found = 0
+        try:
+            with open(self.file_path, 'rb') as f:
+                # Try to parse as XEL binary format
+                for event in self._parse_binary_format(f):
+                    self.events_parsed += 1
+                    events_found += 1
+                    yield event
+
+            # If no events found in binary, try XML fallback
+            if events_found == 0:
+                for event in self._parse_xml_fallback():
+                    self.events_parsed += 1
+                    yield event
+        except Exception as e:
+            # If all parsing fails, try XML parsing as last resort
+            print(f"Parsing error, attempting XML fallback: {e}", file=sys.stderr)
+            try:
+                for event in self._parse_xml_fallback():
+                    self.events_parsed += 1
+                    yield event
+            except Exception as xml_error:
+                raise Exception(f"Failed to parse XEL file: Error: {e}, XML error: {xml_error}")
+
+    def _parse_binary_format(self, f: BinaryIO) -> Iterator[Dict[str, Any]]:
+        """
+        Parse XEL file in binary format.
+
+        XEL files contain a header followed by event blocks. SQL Server XEL files
+        use a proprietary binary format that requires specialized parsing.
+        """
+        # Read all data
+        f.seek(0)
+        data = f.read()
+
+        # Check for XEL magic number
+        if len(data) < 4:
+            return
+
+        # Check if it's an XML file
+        if data[:5] == b'<?xml' or data[:6] == b'<event' or data[:7] == b'<Events':
+            # It's XML, don't parse as binary
+            return
+
+        magic = data[0:4]
+        # Common XEL magic numbers
+        if magic not in [b'Z7\xab\xef', b'\x5a\x37\xab\xef']:
+            # Unknown format, try XML parsing as fallback
+            return
+
+        # For SQL Server XEL files, try to extract structured data
+        # These files use a complex binary format with UTF-16 strings
+        events_found = 0
+
+        # Look for XML event patterns (some XEL variants have XML)
+        xml_start_markers = [b'<event', b'<?xml', b'<\x00e\x00v\x00e\x00n\x00t\x00']
+
+        for marker in xml_start_markers:
+            offset = 0
+            while True:
+                pos = data.find(marker, offset)
+                if pos == -1:
+                    break
+
+                # Try to extract XML from this position
+                try:
+                    xml_data = self._extract_xml_from_position(data, pos)
+                    if xml_data:
+                        event = self._parse_event_xml(xml_data)
+                        if event:
+                            events_found += 1
+                            yield event
+                    offset = pos + 1
+                except Exception:
+                    offset = pos + 1
+                    continue
+
+        # If no XML events found, try to extract real audit data from binary
+        if events_found == 0:
+            try:
+                from .real_parser import extract_real_data_from_xel
+                real_data = extract_real_data_from_xel(self.file_path)
+
+                # Create events from extracted data
+                info = real_data['extracted_info']
+
+                # Create main audit summary event
+                yield {
+                    'name': 'audit_data_extracted',
+                    'timestamp': info.get('timestamps', [''])[0] if info.get('timestamps') else '',
+                    'data': {
+                        'format': 'SQL Server XEL binary format',
+                        'extraction_method': 'Binary data analysis',
+                        'server': info['server_names'][0] if info['server_names'] else 'unknown',
+                        'databases': info['database_names'],
+                        'ip_addresses': info['ip_addresses'],
+                        'event_types': info['event_types'],
+                        'applications': info['applications'],
+                        'users': info['users'],
+                        'file_size_bytes': len(data),
+                    },
+                    'actions': info.get('session_info', {})
+                }
+
+                # Create individual events for reconstructed data
+                for event in real_data.get('reconstructed_events', [])[:10]:
+                    yield {
+                        'name': event.get('event_type', 'unknown'),
+                        'timestamp': event.get('possible_timestamp', ''),
+                        'data': {k: v for k, v in event.items() if k not in ['event_type', 'possible_timestamp']},
+                        'actions': {}
+                    }
+
+            except ImportError:
+                # Fallback to simple string extraction
+                strings = self._extract_utf16_strings(data)
+                yield {
+                    'name': 'binary_data',
+                    'timestamp': '',
+                    'data': {
+                        'format': 'SQL Server XEL binary format',
+                        'note': 'Install xel_real_parser for detailed extraction',
+                        'extracted_strings_count': len(strings),
+                        'extracted_strings_sample': strings[:10] if len(strings) > 10 else strings,
+                        'file_size_bytes': len(data),
+                        'magic_bytes': magic.hex()
+                    },
+                    'actions': {}
+                }
+
+    def _extract_utf16_strings(self, data: bytes, min_length: int = 8) -> list:
+        """
+        Extract UTF-16 encoded strings from binary data.
+
+        Args:
+            data: Binary data
+            min_length: Minimum string length to extract
+
+        Returns:
+            List of extracted strings
+        """
+        strings = []
+        seen = set()
+        try:
+            # Try UTF-16 LE decoding
+            decoded = data.decode('utf-16-le', errors='ignore')
+            # Split on null characters and filter
+            parts = decoded.split('\x00')
+            for part in parts:
+                part = part.strip()
+                # Filter for readable ASCII strings and avoid duplicates
+                if (len(part) >= min_length and
+                    part.isprintable() and
+                    part.isascii() and
+                    part not in seen and
+                    (' ' in part or '.' in part or '_' in part)):  # Likely meaningful text
+                    strings.append(part)
+                    seen.add(part)
+        except:
+            pass
+
+        return list(set(strings))[:100]  # Limit to unique 100 most useful strings
+
+    def _extract_xml_from_position(self, data: bytes, start_pos: int) -> Optional[str]:
+        """
+        Extract complete XML from a starting position in binary data.
+
+        Args:
+            data: Binary data
+            start_pos: Starting position of XML
+
+        Returns:
+            XML string or None if extraction fails
+        """
+        # Look for XML end tag
+        end_markers = [b'</event>', b'</Event>']
+
+        for end_marker in end_markers:
+            end_pos = data.find(end_marker, start_pos)
+            if end_pos != -1:
+                end_pos += len(end_marker)
+                xml_bytes = data[start_pos:end_pos]
+                try:
+                    # Try to decode as UTF-8 or UTF-16
+                    for encoding in ['utf-8', 'utf-16-le', 'utf-16-be', 'latin-1']:
+                        try:
+                            xml_str = xml_bytes.decode(encoding)
+                            # Basic validation
+                            if '<event' in xml_str.lower() and '</event>' in xml_str.lower():
+                                return xml_str
+                        except:
+                            continue
+                except:
+                    pass
+        return None
+
+    def _parse_xml_fallback(self) -> Iterator[Dict[str, Any]]:
+        """
+        Fallback method to parse XEL file as XML.
+        Some tools export XEL data as XML.
+        """
+        try:
+            tree = ET.parse(self.file_path)
+            root = tree.getroot()
+
+            # Handle different XML structures
+            events = root.findall('.//event') or root.findall('.//Event')
+            for event_elem in events:
+                event = self._parse_event_element(event_elem)
+                if event:
+                    yield event
+        except Exception as e:
+            raise Exception(f"XML parsing failed: {e}")
+
+    def _parse_event_xml(self, xml_str: str) -> Optional[Dict[str, Any]]:
+        """
+        Parse an individual event XML string.
+
+        Args:
+            xml_str: XML string containing event data
+
+        Returns:
+            Dictionary with event data or None
+        """
+        try:
+            # Clean up XML string
+            xml_str = xml_str.strip()
+
+            # Parse XML
+            root = ET.fromstring(xml_str)
+            return self._parse_event_element(root)
+        except Exception:
+            return None
+
+    def _parse_event_element(self, elem: ET.Element) -> Dict[str, Any]:
+        """
+        Parse an event XML element into a dictionary.
+
+        Args:
+            elem: XML element representing an event
+
+        Returns:
+            Dictionary with event data
+        """
+        event = {
+            'name': elem.get('name', 'unknown'),
+            'timestamp': elem.get('timestamp', ''),
+            'data': {},
+            'actions': {}
+        }
+
+        # Parse data fields
+        for data_elem in elem.findall('.//data') or elem.findall('.//Data'):
+            name = data_elem.get('name', '')
+            value = self._extract_value(data_elem)
+            if name:
+                event['data'][name] = value
+
+        # Parse action fields
+        for action_elem in elem.findall('.//action') or elem.findall('.//Action'):
+            name = action_elem.get('name', '')
+            value = self._extract_value(action_elem)
+            if name:
+                event['actions'][name] = value
+
+        # Parse any direct text content
+        if elem.text and elem.text.strip():
+            event['content'] = elem.text.strip()
+
+        return event
+
+    def _extract_value(self, elem: ET.Element) -> Any:
+        """
+        Extract value from an XML element.
+
+        Args:
+            elem: XML element
+
+        Returns:
+            Extracted value (string, int, float, etc.)
+        """
+        # Try 'value' attribute first
+        value = elem.get('value')
+        if value is not None:
+            return self._convert_value(value)
+
+        # Try text content
+        if elem.text:
+            return self._convert_value(elem.text.strip())
+
+        # Try child elements
+        value_elem = elem.find('value') or elem.find('Value')
+        if value_elem is not None and value_elem.text:
+            return self._convert_value(value_elem.text.strip())
+
+        return None
+
+    def _convert_value(self, value: str) -> Any:
+        """
+        Convert string value to appropriate type.
+
+        Args:
+            value: String value
+
+        Returns:
+            Converted value
+        """
+        if not value:
+            return value
+
+        # Try integer
+        try:
+            return int(value)
+        except ValueError:
+            pass
+
+        # Try float
+        try:
+            return float(value)
+        except ValueError:
+            pass
+
+        # Try boolean
+        if value.lower() in ('true', 'false'):
+            return value.lower() == 'true'
+
+        # Return as string
+        return value
+
+    def get_stats(self) -> Dict[str, Any]:
+        """
+        Get parsing statistics.
+
+        Returns:
+            Dictionary with stats
+        """
+        return {
+            'file_path': self.file_path,
+            'events_parsed': self.events_parsed
+        }

sql_xel_parser/real_parser.py

@@ -0,0 +1,295 @@
+"""
+Real XEL Parser - Extract actual data from SQL Server binary XEL files.
+
+This parser extracts meaningful audit information from SQL Server Extended Events files
+without requiring SQL Server.
+"""
+
+import struct
+import re
+from typing import List, Dict, Any
+from collections import defaultdict
+
+
+def extract_real_data_from_xel(file_path: str) -> Dict[str, Any]:
+    """
+    Extract real audit/event data from a SQL Server XEL binary file.
+
+    Args:
+        file_path: Path to .xel file
+
+    Returns:
+        Dictionary with extracted event data
+    """
+    with open(file_path, 'rb') as f:
+        data = f.read()
+
+    result = {
+        'file_path': file_path,
+        'file_size': len(data),
+        'magic': data[:4].hex() if len(data) >= 4 else None,
+        'extracted_info': {}
+    }
+
+    # Decode as UTF-16 to extract strings
+    try:
+        decoded = data.decode('utf-16-le', errors='ignore')
+
+        # Extract various types of information
+        result['extracted_info'] = {
+            'server_names': extract_servers(decoded),
+            'database_names': extract_databases(decoded),
+            'sql_statements': extract_sql(decoded),
+            'ip_addresses': extract_ips(decoded),
+            'event_types': extract_event_types(decoded),
+            'timestamps': extract_timestamps(decoded),
+            'session_info': extract_sessions(decoded),
+            'users': extract_users(decoded),
+            'applications': extract_applications(decoded),
+        }
+
+        # Try to reconstruct partial events from the data
+        result['reconstructed_events'] = reconstruct_events(decoded, data)
+
+    except Exception as e:
+        result['error'] = str(e)
+
+    return result
+
+
+def extract_servers(text: str) -> List[str]:
+    """Extract server names."""
+    pattern = r'sql-[a-zA-Z0-9\-]+-[a-z]+\d+'
+    servers = re.findall(pattern, text)
+    return list(set(servers))
+
+
+def extract_databases(text: str) -> List[str]:
+    """Extract database names."""
+    # Look for common database patterns
+    databases = set()
+
+    # Pattern 1: database names in context
+    patterns = [
+        r'(?:database|db)[:\s]+([a-zA-Z][a-zA-Z0-9_]{2,30})',
+        r'USE\s+([a-zA-Z][a-zA-Z0-9_]{2,30})',
+        r'FROM\s+([a-zA-Z][a-zA-Z0-9_]{2,30})\.',
+    ]
+
+    for pattern in patterns:
+        matches = re.findall(pattern, text, re.IGNORECASE)
+        databases.update(matches)
+
+    # Pattern 2: Look for specific known database names
+    # Common patterns in audit logs
+    words = text.split()
+    for i, word in enumerate(words):
+        # Look for DB-like words
+        if len(word) > 3 and word.isalnum() and not word.isdigit():
+            # Check if it appears in database-related context
+            if i > 0 and any(kw in words[i-1].lower() for kw in ['database', 'db', 'use', 'from']):
+                databases.add(word)
+
+    # Common system databases
+    system_dbs = {'master', 'tempdb', 'model', 'msdb'}
+    found_system = [db for db in system_dbs if db in text.lower()]
+    databases.update(found_system)
+
+    # Filter out noise
+    filtered = {db for db in databases if 2 < len(db) < 50 and not db.isdigit()}
+    return sorted(filtered)
+
+
+def extract_sql(text: str) -> List[str]:
+    """Extract SQL statements."""
+    statements = []
+
+    # Look for SQL keywords
+    sql_pattern = r'(SELECT|INSERT|UPDATE|DELETE|EXECUTE|EXEC|CREATE|ALTER|DROP)[\s\S]{10,200}?(?:;|FROM|WHERE|INTO|SET|VALUES)'
+    matches = re.findall(sql_pattern, text, re.IGNORECASE)
+
+    for match in matches:
+        stmt = ' '.join(match.split())  # Normalize whitespace
+        if len(stmt) > 15:
+            statements.append(stmt[:200])  # Limit length
+
+    return list(set(statements))[:20]  # Return up to 20 unique statements
+
+
+def extract_ips(text: str) -> List[str]:
+    """Extract IP addresses."""
+    pattern = r'\b(?:\d{1,3}\.){3}\d{1,3}\b'
+    ips = re.findall(pattern, text)
+    # Filter valid IPs
+    valid_ips = []
+    for ip in ips:
+        parts = ip.split('.')
+        if all(0 <= int(p) <= 255 for p in parts):
+            valid_ips.append(ip)
+    return list(set(valid_ips))
+
+
+def extract_event_types(text: str) -> List[str]:
+    """Extract event types."""
+    event_types = set()
+
+    # Known XEL event types
+    known_types = [
+        'audit_event', 'sql_batch_completed', 'sql_batch_starting',
+        'rpc_completed', 'rpc_starting', 'login', 'logout',
+        'attention', 'existing_connection', 'session_id',
+        'audit_schema_version', 'event_sequence',
+    ]
+
+    for event_type in known_types:
+        if event_type in text.lower():
+            event_types.add(event_type)
+
+    # Look for patterns like "event_*" or "*_event"
+    pattern = r'\b\w+_event\b|\bevent_\w+\b'
+    matches = re.findall(pattern, text, re.IGNORECASE)
+    event_types.update(m.lower() for m in matches)
+
+    return sorted(event_types)
+
+
+def extract_timestamps(text: str) -> List[str]:
+    """Extract timestamps."""
+    timestamps = []
+
+    # ISO format timestamps
+    pattern = r'\d{4}-\d{2}-\d{2}[T\s]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?'
+    matches = re.findall(pattern, text)
+    timestamps.extend(matches)
+
+    return list(set(timestamps))[:50]  # Limit to 50
+
+
+def extract_sessions(text: str) -> Dict[str, Any]:
+    """Extract session information."""
+    session_info = {}
+
+    # Look for session IDs
+    pattern = r'session[_\s]*id[:\s]*(\d+)'
+    matches = re.findall(pattern, text, re.IGNORECASE)
+    if matches:
+        session_info['session_ids'] = list(set(matches))[:20]
+
+    # Look for SqlDbAuditing sessions
+    pattern = r'SqlDbAuditing[^<>]{0,100}'
+    matches = re.findall(pattern, text)
+    if matches:
+        session_info['audit_sessions'] = list(set(matches))[:10]
+
+    return session_info
+
+
+def extract_users(text: str) -> List[str]:
+    """Extract usernames."""
+    users = set()
+
+    # Look for common user patterns
+    patterns = [
+        r'user[:\s]+([a-zA-Z][a-zA-Z0-9_@\-\.]{2,50})',
+        r'login[:\s]+([a-zA-Z][a-zA-Z0-9_@\-\.]{2,50})',
+        r'USER_NAME\(\)[:\s]*([a-zA-Z][a-zA-Z0-9_@\-\.]{2,50})',
+    ]
+
+    for pattern in patterns:
+        matches = re.findall(pattern, text, re.IGNORECASE)
+        users.update(matches)
+
+    # Common system users
+    if 'sa' in text and 'sa' not in users:
+        users.add('sa')
+    if 'dbo' in text and 'dbo' not in users:
+        users.add('dbo')
+
+    # Filter out noise
+    filtered = {u for u in users if 2 < len(u) < 100}
+    return sorted(filtered)[:30]
+
+
+def extract_applications(text: str) -> List[str]:
+    """Extract application names."""
+    apps = set()
+
+    # Look for common application patterns
+    patterns = [
+        r'Core Microsoft SqlClient Data Provider',
+        r'\.NET [^<>\s]{5,50}',
+        r'ODBC Driver \d+',
+        r'[A-Z][a-zA-Z0-9\s]{5,50}(?:Application|App|Client|Service)',
+    ]
+
+    for pattern in patterns:
+        matches = re.findall(pattern, text)
+        apps.update(matches)
+
+    return list(apps)[:20]
+
+
+def reconstruct_events(text: str, binary_data: bytes) -> List[Dict[str, Any]]:
+    """
+    Attempt to reconstruct event records from extracted data.
+    """
+    events = []
+
+    # Strategy: Look for combinations of extracted data that likely represent events
+
+    # Extract all the pieces
+    servers = extract_servers(text)
+    databases = extract_databases(text)
+    sql_stmts = extract_sql(text)
+    ips = extract_ips(text)
+    timestamps = extract_timestamps(text)
+    event_types = extract_event_types(text)
+
+    # If we have SQL statements, create events for them
+    for i, stmt in enumerate(sql_stmts):
+        event = {
+            'event_type': 'sql_execution',
+            'statement': stmt,
+        }
+
+        # Try to associate with other data
+        if databases:
+            event['possible_database'] = databases[i % len(databases)]
+        if servers:
+            event['server'] = servers[0]
+        if timestamps and i < len(timestamps):
+            event['possible_timestamp'] = timestamps[i]
+
+        events.append(event)
+
+    # Create generic audit events for other extracted info
+    if event_types:
+        for event_type in event_types:
+            event = {
+                'event_type': event_type,
+                'extracted_from': 'audit_metadata'
+            }
+            if servers:
+                event['server'] = servers[0]
+            events.append(event)
+
+    return events[:50]  # Limit to 50 events
+
+
+def parse_xel_file(file_path: str) -> Dict[str, Any]:
+    """
+    Main entry point to parse a real XEL file.
+    """
+    return extract_real_data_from_xel(file_path)
+
+
+if __name__ == '__main__':
+    import sys
+    import json
+
+    if len(sys.argv) < 2:
+        print("Usage: python xel_real_parser.py <xel_file>")
+        sys.exit(1)
+
+    result = parse_xel_file(sys.argv[1])
+    print(json.dumps(result, indent=2))