quantum-framework 0.18.0__py3-none-any.whl

quantum/core/features/data_import/src/runtime.py

@@ -0,0 +1,533 @@
+"""
+Data Import Feature - Runtime
+Handles execution of all data import and transformation operations
+"""
+
+import time
+import json
+import csv
+import requests
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Dict, Any, Optional, List
+from dataclasses import dataclass
+from io import StringIO
+import re
+
+from quantum.core.expressions import ExpressionEvaluator
+
+# Shared with ComponentRuntime and HTMLRenderer, so a {field} expression in a
+# q:data filter means the same thing it means anywhere else in the language.
+_EXPRESSIONS = ExpressionEvaluator()
+_BINDING = re.compile(r'\{([^}]+)\}')
+
+
+@dataclass
+class DataResult:
+    """Result object for all data operations"""
+    success: bool
+    data: Any = None
+    error: Optional[Dict[str, Any]] = None
+    recordCount: int = 0
+    loadTime: float = 0.0
+    cached: bool = False
+    source: str = ""
+
+
+class DataImportService:
+    """Service to handle all types of data import and transformation"""
+
+    def __init__(self):
+        self.cache: Dict[str, Any] = {}  # Simple in-memory cache
+
+    def import_data(
+        self,
+        data_type: str,
+        source: str,
+        params: Dict[str, Any],
+        context: Any = None
+    ) -> DataResult:
+        """Main data import dispatcher"""
+        start_time = time.time()
+
+        try:
+            # Check cache if enabled
+            cache_enabled = params.get('cache', True)
+            if cache_enabled:
+                cache_key = f"data_{source}_{hash(str(params))}"
+                cached_result = self.get_from_cache(cache_key)
+                if cached_result is not None:
+                    cached_result.cached = True
+                    return cached_result
+
+            # Dispatch based on type
+            if data_type == "csv":
+                result = self._import_csv(source, params)
+            elif data_type == "json":
+                result = self._import_json(source, params)
+            elif data_type == "xml":
+                result = self._import_xml(source, params)
+            elif data_type == "transform":
+                result = self._transform_data(source, params, context)
+            else:
+                return DataResult(
+                    success=False,
+                    error={"message": f"Unknown data type: {data_type}"},
+                    source=source
+                )
+
+            # Calculate load time
+            load_time = (time.time() - start_time) * 1000  # Convert to ms
+            result.loadTime = load_time
+
+            # Apply transformations if specified
+            if 'transforms' in params and params['transforms']:
+                result = self._apply_transformations(result, params['transforms'], context)
+
+            # Cache result if enabled
+            if cache_enabled and result.success:
+                ttl = params.get('ttl')
+                self.put_in_cache(cache_key, result, ttl)
+
+            return result
+
+        except Exception as e:
+            load_time = (time.time() - start_time) * 1000
+            return DataResult(
+                success=False,
+                error={
+                    "message": str(e),
+                    "type": type(e).__name__
+                },
+                loadTime=load_time,
+                source=source
+            )
+
+    def _import_csv(self, source: str, params: Dict[str, Any]) -> DataResult:
+        """Import CSV data from file or URL"""
+        try:
+            # Get CSV content
+            content = self._get_source_content(source, params.get('headers', []))
+
+            # Parse CSV
+            delimiter = params.get('delimiter', ',')
+            quote = params.get('quote', '"')
+            has_header = params.get('header', True)
+            encoding = params.get('encoding', 'utf-8')
+            skip_rows = params.get('skip_rows', 0)
+            columns = params.get('columns', [])
+
+            # Parse CSV content
+            csv_reader = csv.DictReader(
+                StringIO(content),
+                delimiter=delimiter,
+                quotechar=quote
+            ) if has_header else csv.reader(StringIO(content), delimiter=delimiter, quotechar=quote)
+
+            # Skip rows if needed
+            for _ in range(skip_rows):
+                try:
+                    next(csv_reader)
+                except StopIteration:
+                    break
+
+            # Convert to list of dictionaries
+            data = []
+            for row in csv_reader:
+                if has_header:
+                    # Row is already a dict
+                    record = self._convert_csv_row(row, columns)
+                else:
+                    # Row is a list - create dict from column definitions
+                    if columns:
+                        record = {}
+                        for i, col_def in enumerate(columns):
+                            if i < len(row):
+                                col_name = col_def.get('name')
+                                col_value = row[i]
+                                record[col_name] = self._convert_type(col_value, col_def.get('type', 'string'))
+                    else:
+                        # No columns defined - use numeric keys
+                        record = {str(i): val for i, val in enumerate(row)}
+
+                data.append(record)
+
+            return DataResult(
+                success=True,
+                data=data,
+                recordCount=len(data),
+                source=source
+            )
+
+        except Exception as e:
+            return DataResult(
+                success=False,
+                error={
+                    "message": f"CSV import failed: {str(e)}",
+                    "type": type(e).__name__
+                },
+                source=source
+            )
+
+    def _import_json(self, source: str, params: Dict[str, Any]) -> DataResult:
+        """Import JSON data from file or URL"""
+        try:
+            # Get JSON content
+            content = self._get_source_content(source, params.get('headers', []))
+
+            # Parse JSON
+            data = json.loads(content)
+
+            # Ensure data is a list
+            if not isinstance(data, list):
+                # If it's a dict, wrap it in a list
+                data = [data]
+
+            return DataResult(
+                success=True,
+                data=data,
+                recordCount=len(data) if isinstance(data, list) else 1,
+                source=source
+            )
+
+        except json.JSONDecodeError as e:
+            return DataResult(
+                success=False,
+                error={
+                    "message": f"Invalid JSON: {str(e)}",
+                    "type": "JSONDecodeError"
+                },
+                source=source
+            )
+        except Exception as e:
+            return DataResult(
+                success=False,
+                error={
+                    "message": f"JSON import failed: {str(e)}",
+                    "type": type(e).__name__
+                },
+                source=source
+            )
+
+    def _import_xml(self, source: str, params: Dict[str, Any]) -> DataResult:
+        """Import XML data from file or URL with XPath"""
+        try:
+            # Get XML content
+            content = self._get_source_content(source, params.get('headers', []))
+
+            # Parse XML
+            root = ET.fromstring(content)
+
+            # Get XPath expression
+            xpath = params.get('xpath', '.')
+            fields = params.get('fields', [])
+
+            # Find nodes using XPath (basic support - ET has limited XPath)
+            nodes = root.findall(xpath)
+
+            # Extract data from nodes
+            data = []
+            for node in nodes:
+                record = {}
+                for field_def in fields:
+                    field_name = field_def.get('name')
+                    field_xpath = field_def.get('xpath')
+                    field_type = field_def.get('type', 'string')
+
+                    # Extract value using XPath
+                    value = self._extract_xml_value(node, field_xpath)
+                    record[field_name] = self._convert_type(value, field_type)
+
+                data.append(record)
+
+            return DataResult(
+                success=True,     # was False: every XML import "failed", so no transform ran on it
+                data=data,
+                recordCount=len(data),
+                source=source
+            )
+
+        except ET.ParseError as e:
+            return DataResult(
+                success=False,
+                error={
+                    "message": f"Invalid XML: {str(e)}",
+                    "type": "XMLParseError"
+                },
+                source=source
+            )
+        except Exception as e:
+            return DataResult(
+                success=False,
+                error={
+                    "message": f"XML import failed: {str(e)}",
+                    "type": type(e).__name__
+                },
+                source=source
+            )
+
+    def _transform_data(self, source: str, params: Dict[str, Any], context: Any) -> DataResult:
+        """Transform existing data (from variable reference)"""
+        try:
+            # Source should be a variable reference like "{users}"
+            # Extract variable name and get data from context
+            if source.startswith('{') and source.endswith('}'):
+                var_name = source[1:-1]  # Remove braces
+            else:
+                var_name = source
+
+            # Get data from context
+            if hasattr(context, 'get_variable'):
+                data = context.get_variable(var_name)
+            elif isinstance(context, dict):
+                data = context.get(var_name)
+            else:
+                return DataResult(
+                    success=False,
+                    error={"message": f"Cannot access variable '{var_name}' from context"},
+                    source=source
+                )
+
+            if data is None:
+                return DataResult(
+                    success=False,
+                    error={"message": f"Variable '{var_name}' not found"},
+                    source=source
+                )
+
+            # Ensure data is a list
+            if not isinstance(data, list):
+                data = [data]
+
+            return DataResult(
+                success=True,
+                data=data,
+                recordCount=len(data),
+                source=source
+            )
+
+        except Exception as e:
+            return DataResult(
+                success=False,
+                error={
+                    "message": f"Transform failed: {str(e)}",
+                    "type": type(e).__name__
+                },
+                source=source
+            )
+
+    def _apply_transformations(self, result: DataResult, transforms: List[Dict[str, Any]], context: Any) -> DataResult:
+        """Apply transformation operations to data"""
+        if not result.success or not result.data:
+            return result
+
+        data = result.data
+
+        for transform_op in transforms:
+            op_type = transform_op.get('type')
+
+            if op_type == 'filter':
+                data = self._apply_filter(data, transform_op, context)
+            elif op_type == 'sort':
+                data = self._apply_sort(data, transform_op)
+            elif op_type == 'limit':
+                data = self._apply_limit(data, transform_op)
+            elif op_type == 'compute':
+                data = self._apply_compute(data, transform_op, context)
+
+        # Update result
+        result.data = data
+        result.recordCount = len(data)
+        return result
+
+    def _apply_filter(self, data: List[Dict[str, Any]], filter_op: Dict[str, Any], context: Any) -> List[Dict[str, Any]]:
+        """Filter data based on condition"""
+        condition = filter_op.get('condition', '')
+
+        filtered = []
+        for record in data:
+            # Evaluate condition for this record
+            if self._evaluate_condition(condition, record, context):
+                filtered.append(record)
+
+        return filtered
+
+    def _apply_sort(self, data: List[Dict[str, Any]], sort_op: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """Sort data by field"""
+        by_field = sort_op.get('by')
+        order = sort_op.get('order', 'asc')
+
+        if not by_field:
+            return data
+
+        try:
+            # Sort data
+            reverse = (order == 'desc')
+            sorted_data = sorted(data, key=lambda x: x.get(by_field, ''), reverse=reverse)
+            return sorted_data
+        except Exception:
+            # If sorting fails, return original data
+            return data
+
+    def _apply_limit(self, data: List[Dict[str, Any]], limit_op: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """Limit number of records"""
+        limit = limit_op.get('value', 0)
+
+        if isinstance(limit, str):
+            try:
+                limit = int(limit)
+            except ValueError:
+                return data
+
+        return data[:limit]
+
+    def _apply_compute(self, data: List[Dict[str, Any]], compute_op: Dict[str, Any], context: Any) -> List[Dict[str, Any]]:
+        """Compute derived field"""
+        field = compute_op.get('field')
+        expression = compute_op.get('expression')
+        comp_type = compute_op.get('comp_type', 'string')
+
+        if not field or not expression:
+            return data
+
+        # Add computed field to each record
+        for record in data:
+            value = self._evaluate_expression(expression, record, context)
+            record[field] = self._convert_type(value, comp_type)
+
+        return data
+
+    def _get_source_content(self, source: str, headers: List[Dict[str, str]] = None) -> str:
+        """Get content from file or URL"""
+        # Check if source is a URL
+        if source.startswith('http://') or source.startswith('https://'):
+            # Fetch from URL
+            headers_dict = {}
+            if headers:
+                for header in headers:
+                    headers_dict[header.get('name')] = header.get('value')
+
+            response = requests.get(source, headers=headers_dict, timeout=30)
+            response.raise_for_status()
+            return response.text
+        else:
+            # Read from file
+            file_path = Path(source)
+            if not file_path.is_absolute():
+                # Relative path - resolve from current working directory
+                file_path = Path.cwd() / file_path
+
+            with open(file_path, 'r', encoding='utf-8') as f:
+                return f.read()
+
+    def _convert_csv_row(self, row: Dict[str, str], columns: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """Convert CSV row values to proper types"""
+        if not columns:
+            return row
+
+        converted = {}
+        for col_def in columns:
+            col_name = col_def.get('name')
+            col_type = col_def.get('type', 'string')
+
+            if col_name in row:
+                converted[col_name] = self._convert_type(row[col_name], col_type)
+
+        # Include any columns not defined in schema
+        for key, value in row.items():
+            if key not in converted:
+                converted[key] = value
+
+        return converted
+
+    def _convert_type(self, value: Any, target_type: str) -> Any:
+        """Convert value to target type"""
+        if value is None or value == '':
+            return None
+
+        try:
+            if target_type == 'integer':
+                return int(value)
+            elif target_type == 'decimal':
+                return float(value)
+            elif target_type == 'boolean':
+                if isinstance(value, bool):
+                    return value
+                if isinstance(value, str):
+                    return value.lower() in ('true', '1', 'yes', 'on')
+                return bool(value)
+            elif target_type == 'json':
+                if isinstance(value, str):
+                    return json.loads(value)
+                return value
+            elif target_type == 'array':
+                if isinstance(value, list):
+                    return value
+                if isinstance(value, str):
+                    return json.loads(value)
+                return [value]
+            else:  # string
+                return str(value)
+        except (ValueError, json.JSONDecodeError):
+            return value
+
+    def _extract_xml_value(self, node: ET.Element, xpath: str) -> Optional[str]:
+        """Extract value from XML node using XPath"""
+        # Handle attribute access (@attr)
+        if xpath.startswith('@'):
+            attr_name = xpath[1:]
+            return node.get(attr_name)
+
+        # Handle text content (text())
+        if xpath == 'text()':
+            return node.text
+
+        # child/@attr — ElementTree's find() does not understand the @ step.
+        if '/@' in xpath:
+            child_path, attr = xpath.rsplit('/@', 1)
+            child = node.find(child_path)
+            return child.get(attr) if child is not None else None
+
+        # child/text() — nor text(); it raised SyntaxError on every such field.
+        if xpath.endswith('/text()'):
+            xpath = xpath[:-len('/text()')]
+
+        child = node.find(xpath)
+        return child.text if child is not None else None
+
+    def _evaluate_condition(self, condition: str, record: Dict[str, Any], context: Any) -> bool:
+        """Evaluate a q:data filter condition against one imported record.
+
+        `{field}` names the record's field. The braces are stripped so the
+        evaluator resolves the NAME; the record's values are never spliced
+        into the expression text.
+
+        That distinction is the whole fix. This used to build a string by
+        interpolating each field value into the condition — quoting strings as
+        f"'{value}'" — and then call a bare eval() on it, with full builtins.
+        A single quote anywhere in an imported CSV cell closed the literal and
+        the rest of the cell ran as Python. The input here is an imported file,
+        which is the least trusted thing the framework touches.
+        """
+        return bool(self._evaluate_expression(condition, record, context))
+
+    def _evaluate_expression(self, expression: str, record: Dict[str, Any], context: Any) -> Any:
+        """Evaluate a computed-field expression against one imported record.
+
+        See _evaluate_condition for why this no longer interpolates values.
+        """
+        expr = _BINDING.sub(lambda m: m.group(1), expression).strip()
+        return _EXPRESSIONS.try_evaluate(expr, record, default=None)
+
+    def get_from_cache(self, cache_key: str) -> Optional[DataResult]:
+        """Get value from cache"""
+        return self.cache.get(cache_key)
+
+    def put_in_cache(self, cache_key: str, value: DataResult, ttl: Optional[int] = None):
+        """Put value in cache"""
+        # Simple implementation - in production you'd use TTL properly
+        self.cache[cache_key] = value
+
+    def clear_cache(self):
+        """Clear all cache"""
+        self.cache.clear()

quantum/core/features/dump/src/__init__.py

@@ -0,0 +1,12 @@
+"""
+Dump feature for Quantum Language
+
+Provides beautiful variable inspection for debugging,
+like ColdFusion's cfdump but better.
+"""
+
+from .ast_node import DumpNode
+from .parser import parse_dump
+from .runtime import DumpService
+
+__all__ = ['DumpNode', 'parse_dump', 'DumpService']

quantum/core/features/dump/src/ast_node.py

@@ -0,0 +1,106 @@
+"""
+AST Node for q:dump - Variable Inspection
+"""
+
+from typing import Optional
+from quantum.core.ast_nodes import QuantumNode
+
+
+class DumpNode(QuantumNode):
+    """
+    Represents a <q:dump> - Variable inspection statement
+
+    Example:
+        <q:dump var="{user}" label="User Object" />
+        <q:dump var="{data}" format="json" depth="3" />
+    """
+
+    VALID_FORMATS = ['html', 'json', 'text']
+
+    def __init__(
+        self,
+        var: str,
+        label: Optional[str] = None,
+        format: str = 'html',
+        depth: int = 10,
+        when: Optional[str] = None,
+        expand: int = 2,
+        show_types: bool = True
+    ):
+        """
+        Initialize a DumpNode
+
+        Args:
+            var: Variable to inspect (databinding expression)
+            label: Optional identifying label
+            format: Output format (html, json, text)
+            depth: Max depth for nested structures
+            when: Optional conditional expression
+            expand: Auto-expand depth (Phase 2)
+            show_types: Show type information (Phase 2)
+        """
+        # Validate var
+        if not var:
+            raise ValueError("Dump requires 'var' attribute")
+
+        # Validate format
+        if format and format not in self.VALID_FORMATS:
+            raise ValueError(
+                f"Invalid dump format '{format}'. Must be one of: {', '.join(self.VALID_FORMATS)}"
+            )
+
+        # Validate depth
+        if depth is not None:
+            try:
+                depth_int = int(depth)
+                if depth_int < 0:
+                    raise ValueError("Dump depth must be a positive integer")
+                depth = depth_int
+            except (ValueError, TypeError):
+                raise ValueError("Dump depth must be a positive integer")
+
+        self.var = var
+        self.label = label if label else var
+        self.format = format
+        self.depth = depth
+        self.when = when
+        self.expand = expand
+        self.show_types = show_types
+
+    def __repr__(self):
+        attrs = [f"var={self.var!r}"]
+        if self.label != self.var:
+            attrs.append(f"label={self.label!r}")
+        if self.format != 'html':
+            attrs.append(f"format={self.format!r}")
+        if self.when:
+            attrs.append(f"when={self.when!r}")
+        return f"DumpNode({', '.join(attrs)})"
+
+    def to_dict(self):
+        """Convert node to dictionary for debugging/serialization"""
+        return {
+            "type": "dump",
+            "var": self.var,
+            "label": self.label,
+            "format": self.format,
+            "depth": self.depth,
+            "when": self.when,
+            "expand": self.expand,
+            "show_types": self.show_types
+        }
+
+    def validate(self):
+        """Validate the node and return list of errors"""
+        errors = []
+
+        if not self.var:
+            errors.append("Dump requires 'var' attribute")
+
+        if self.format and self.format not in self.VALID_FORMATS:
+            errors.append(f"Invalid dump format '{self.format}'. Must be one of: {', '.join(self.VALID_FORMATS)}")
+
+        if self.depth is not None and (not isinstance(self.depth, int) or self.depth < 0):
+            errors.append("Dump depth must be a positive integer")
+
+        return errors