genxai-framework 0.1.0__py3-none-any.whl

genxai/tools/builtin/data/csv_processor.py

@@ -0,0 +1,213 @@
+"""CSV processor tool for parsing and manipulating CSV data."""
+
+from typing import Any, Dict, List, Optional
+import logging
+import csv
+import io
+
+from genxai.tools.base import Tool, ToolMetadata, ToolParameter, ToolCategory
+
+logger = logging.getLogger(__name__)
+
+
+class CSVProcessorTool(Tool):
+    """Process, parse, and manipulate CSV data."""
+
+    def __init__(self) -> None:
+        """Initialize CSV processor tool."""
+        metadata = ToolMetadata(
+            name="csv_processor",
+            description="Parse, validate, filter, and transform CSV data",
+            category=ToolCategory.DATA,
+            tags=["csv", "data", "parsing", "tabular", "spreadsheet"],
+            version="1.0.0",
+        )
+
+        parameters = [
+            ToolParameter(
+                name="data",
+                type="string",
+                description="CSV string to process",
+                required=True,
+            ),
+            ToolParameter(
+                name="operation",
+                type="string",
+                description="Operation to perform",
+                required=True,
+                enum=["parse", "filter", "transform", "aggregate", "validate"],
+            ),
+            ToolParameter(
+                name="delimiter",
+                type="string",
+                description="CSV delimiter character",
+                required=False,
+                default=",",
+            ),
+            ToolParameter(
+                name="has_header",
+                type="boolean",
+                description="Whether CSV has header row",
+                required=False,
+                default=True,
+            ),
+            ToolParameter(
+                name="filter_column",
+                type="string",
+                description="Column name to filter by (for filter operation)",
+                required=False,
+            ),
+            ToolParameter(
+                name="filter_value",
+                type="string",
+                description="Value to filter for (for filter operation)",
+                required=False,
+            ),
+        ]
+
+        super().__init__(metadata, parameters)
+
+    async def _execute(
+        self,
+        data: str,
+        operation: str,
+        delimiter: str = ",",
+        has_header: bool = True,
+        filter_column: Optional[str] = None,
+        filter_value: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Execute CSV processing.
+
+        Args:
+            data: CSV string
+            operation: Operation to perform
+            delimiter: CSV delimiter
+            has_header: Has header flag
+            filter_column: Column to filter
+            filter_value: Value to filter
+
+        Returns:
+            Dictionary containing processed data
+        """
+        result: Dict[str, Any] = {
+            "operation": operation,
+            "success": False,
+        }
+
+        try:
+            # Parse CSV
+            csv_file = io.StringIO(data)
+            reader = csv.reader(csv_file, delimiter=delimiter)
+            rows = list(reader)
+
+            if not rows:
+                raise ValueError("Empty CSV data")
+
+            headers = rows[0] if has_header else [f"col_{i}" for i in range(len(rows[0]))]
+            data_rows = rows[1:] if has_header else rows
+
+            if operation == "parse":
+                # Convert to list of dictionaries
+                parsed_data = []
+                for row in data_rows:
+                    if len(row) == len(headers):
+                        parsed_data.append(dict(zip(headers, row)))
+                
+                result["data"] = parsed_data
+                result["headers"] = headers
+                result["row_count"] = len(parsed_data)
+                result["column_count"] = len(headers)
+                result["success"] = True
+
+            elif operation == "filter":
+                if not filter_column or filter_value is None:
+                    raise ValueError("filter_column and filter_value required for filter operation")
+                
+                if filter_column not in headers:
+                    raise ValueError(f"Column '{filter_column}' not found in headers")
+                
+                col_index = headers.index(filter_column)
+                filtered_rows = [
+                    row for row in data_rows
+                    if len(row) > col_index and row[col_index] == filter_value
+                ]
+                
+                filtered_data = [dict(zip(headers, row)) for row in filtered_rows]
+                result["data"] = filtered_data
+                result["filtered_count"] = len(filtered_data)
+                result["original_count"] = len(data_rows)
+                result["success"] = True
+
+            elif operation == "transform":
+                # Convert to structured format
+                transformed = {
+                    "headers": headers,
+                    "rows": data_rows,
+                    "metadata": {
+                        "row_count": len(data_rows),
+                        "column_count": len(headers),
+                        "delimiter": delimiter,
+                    }
+                }
+                result["data"] = transformed
+                result["success"] = True
+
+            elif operation == "aggregate":
+                # Calculate basic statistics for numeric columns
+                aggregates = {}
+                
+                for col_idx, header in enumerate(headers):
+                    column_values = [row[col_idx] for row in data_rows if len(row) > col_idx]
+                    
+                    # Try to convert to numbers
+                    numeric_values = []
+                    for val in column_values:
+                        try:
+                            numeric_values.append(float(val))
+                        except (ValueError, TypeError):
+                            pass
+                    
+                    if numeric_values:
+                        aggregates[header] = {
+                            "count": len(numeric_values),
+                            "sum": sum(numeric_values),
+                            "mean": sum(numeric_values) / len(numeric_values),
+                            "min": min(numeric_values),
+                            "max": max(numeric_values),
+                        }
+                    else:
+                        # For non-numeric columns, count unique values
+                        unique_values = set(column_values)
+                        aggregates[header] = {
+                            "count": len(column_values),
+                            "unique_count": len(unique_values),
+                            "sample_values": list(unique_values)[:5],
+                        }
+                
+                result["data"] = aggregates
+                result["success"] = True
+
+            elif operation == "validate":
+                # Validate CSV structure
+                issues = []
+                
+                # Check for consistent column count
+                expected_cols = len(headers)
+                for idx, row in enumerate(data_rows):
+                    if len(row) != expected_cols:
+                        issues.append({
+                            "row": idx + (2 if has_header else 1),
+                            "issue": f"Expected {expected_cols} columns, found {len(row)}",
+                        })
+                
+                result["valid"] = len(issues) == 0
+                result["issues"] = issues
+                result["row_count"] = len(data_rows)
+                result["column_count"] = len(headers)
+                result["success"] = True
+
+        except Exception as e:
+            result["error"] = str(e)
+
+        logger.info(f"CSV {operation} operation completed: success={result['success']}")
+        return result

genxai/tools/builtin/data/data_transformer.py

@@ -0,0 +1,299 @@
+"""Data transformer tool for converting between data formats."""
+
+from typing import Any, Dict, Optional
+import logging
+import json
+import csv
+import io
+
+from genxai.tools.base import Tool, ToolMetadata, ToolParameter, ToolCategory
+
+logger = logging.getLogger(__name__)
+
+
+class DataTransformerTool(Tool):
+    """Transform data between different formats (JSON, CSV, XML, etc.)."""
+
+    def __init__(self) -> None:
+        """Initialize data transformer tool."""
+        metadata = ToolMetadata(
+            name="data_transformer",
+            description="Convert data between different formats (JSON, CSV, XML, YAML)",
+            category=ToolCategory.DATA,
+            tags=["transformation", "conversion", "format", "data", "json", "csv", "xml"],
+            version="1.0.0",
+        )
+
+        parameters = [
+            ToolParameter(
+                name="data",
+                type="string",
+                description="Input data to transform",
+                required=True,
+            ),
+            ToolParameter(
+                name="operation",
+                type="string",
+                description="Simple text transformation operation (used by unit tests)",
+                required=False,
+                default="uppercase",
+                enum=["uppercase", "lowercase", "trim", "convert"],
+            ),
+            ToolParameter(
+                name="source_format",
+                type="string",
+                description="Source data format (for convert operation)",
+                required=False,
+                default="json",
+                enum=["json", "csv", "xml", "yaml"],
+            ),
+            ToolParameter(
+                name="target_format",
+                type="string",
+                description="Target data format (for convert operation)",
+                required=False,
+                default="json",
+                enum=["json", "csv", "xml", "yaml"],
+            ),
+            ToolParameter(
+                name="csv_delimiter",
+                type="string",
+                description="CSV delimiter (for CSV operations)",
+                required=False,
+                default=",",
+            ),
+        ]
+
+        super().__init__(metadata, parameters)
+
+    async def _execute(
+        self,
+        data: str,
+        operation: str = "convert",
+        source_format: str = "json",
+        target_format: str = "json",
+        csv_delimiter: str = ",",
+    ) -> Dict[str, Any]:
+        """Execute data transformation.
+
+        Args:
+            data: Input data
+            source_format: Source format
+            target_format: Target format
+            csv_delimiter: CSV delimiter
+
+        Returns:
+            Dictionary containing transformed data
+        """
+        result: Dict[str, Any] = {
+            "operation": operation,
+            "success": False,
+        }
+
+        try:
+            if operation == "uppercase":
+                result["result"] = data.upper()
+                result["success"] = True
+                return result
+            if operation == "lowercase":
+                result["result"] = data.lower()
+                result["success"] = True
+                return result
+            if operation == "trim":
+                result["result"] = data.strip()
+                result["success"] = True
+                return result
+
+            if operation not in {"convert"}:
+                raise ValueError(f"Unsupported operation: {operation}")
+
+            # Parse source data
+            parsed_data = self._parse_data(data, source_format, csv_delimiter)
+
+            # Convert to target format
+            transformed_data = self._convert_data(parsed_data, target_format, csv_delimiter)
+
+            result["data"] = transformed_data
+            result["source_format"] = source_format
+            result["target_format"] = target_format
+            result["success"] = True
+
+        except Exception as e:
+            result["error"] = str(e)
+
+        logger.info(
+            f"Data transformer ({operation}) completed: success={result['success']}"
+        )
+        return result
+
+    def _parse_data(self, data: str, format: str, delimiter: str) -> Any:
+        """Parse data from source format.
+
+        Args:
+            data: Input data
+            format: Data format
+            delimiter: CSV delimiter
+
+        Returns:
+            Parsed data
+        """
+        if format == "json":
+            return json.loads(data)
+        
+        elif format == "csv":
+            csv_file = io.StringIO(data)
+            reader = csv.DictReader(csv_file, delimiter=delimiter)
+            return list(reader)
+        
+        elif format == "xml":
+            import xml.etree.ElementTree as ET
+            root = ET.fromstring(data)
+            return self._xml_to_dict(root)
+        
+        elif format == "yaml":
+            try:
+                import yaml
+                return yaml.safe_load(data)
+            except ImportError:
+                raise ImportError(
+                    "pyyaml package not installed. Install with: pip install pyyaml"
+                )
+        
+        else:
+            raise ValueError(f"Unsupported source format: {format}")
+
+    def _convert_data(self, data: Any, format: str, delimiter: str) -> str:
+        """Convert data to target format.
+
+        Args:
+            data: Parsed data
+            format: Target format
+            delimiter: CSV delimiter
+
+        Returns:
+            Converted data string
+        """
+        if format == "json":
+            return json.dumps(data, indent=2)
+        
+        elif format == "csv":
+            if not isinstance(data, list):
+                raise ValueError("CSV conversion requires list of dictionaries")
+            
+            if not data:
+                return ""
+            
+            output = io.StringIO()
+            if isinstance(data[0], dict):
+                writer = csv.DictWriter(output, fieldnames=data[0].keys(), delimiter=delimiter)
+                writer.writeheader()
+                writer.writerows(data)
+            else:
+                writer = csv.writer(output, delimiter=delimiter)
+                writer.writerows(data)
+            
+            return output.getvalue()
+        
+        elif format == "xml":
+            import xml.etree.ElementTree as ET
+            root = self._dict_to_xml(data, "root")
+            self._indent_xml(root)
+            return ET.tostring(root, encoding="unicode")
+        
+        elif format == "yaml":
+            try:
+                import yaml
+                return yaml.dump(data, default_flow_style=False, sort_keys=False)
+            except ImportError:
+                raise ImportError(
+                    "pyyaml package not installed. Install with: pip install pyyaml"
+                )
+        
+        else:
+            raise ValueError(f"Unsupported target format: {format}")
+
+    def _xml_to_dict(self, element: Any) -> Dict[str, Any]:
+        """Convert XML element to dictionary.
+
+        Args:
+            element: XML element
+
+        Returns:
+            Dictionary representation
+        """
+        result: Dict[str, Any] = {}
+        
+        # Add attributes
+        if element.attrib:
+            result["@attributes"] = element.attrib
+        
+        # Add text content
+        if element.text and element.text.strip():
+            result["#text"] = element.text.strip()
+        
+        # Add children
+        for child in element:
+            child_data = self._xml_to_dict(child)
+            if child.tag in result:
+                # Multiple children with same tag - convert to list
+                if not isinstance(result[child.tag], list):
+                    result[child.tag] = [result[child.tag]]
+                result[child.tag].append(child_data)
+            else:
+                result[child.tag] = child_data
+        
+        return {element.tag: result} if result else {element.tag: element.text}
+
+    def _dict_to_xml(self, data: Any, root_tag: str = "root") -> Any:
+        """Convert dictionary to XML element.
+
+        Args:
+            data: Dictionary data
+            root_tag: Root element tag
+
+        Returns:
+            XML element
+        """
+        import xml.etree.ElementTree as ET
+        
+        root = ET.Element(root_tag)
+        
+        if isinstance(data, dict):
+            for key, value in data.items():
+                if key == "@attributes":
+                    root.attrib.update(value)
+                elif key == "#text":
+                    root.text = str(value)
+                else:
+                    if isinstance(value, list):
+                        for item in value:
+                            child = self._dict_to_xml(item, key)
+                            root.append(child)
+                    else:
+                        child = self._dict_to_xml(value, key)
+                        root.append(child)
+        else:
+            root.text = str(data)
+        
+        return root
+
+    def _indent_xml(self, elem: Any, level: int = 0) -> None:
+        """Add indentation to XML element.
+
+        Args:
+            elem: XML element
+            level: Indentation level
+        """
+        indent = "\n" + "  " * level
+        if len(elem):
+            if not elem.text or not elem.text.strip():
+                elem.text = indent + "  "
+            if not elem.tail or not elem.tail.strip():
+                elem.tail = indent
+            for child in elem:
+                self._indent_xml(child, level + 1)
+            if not child.tail or not child.tail.strip():
+                child.tail = indent
+        else:
+            if level and (not elem.tail or not elem.tail.strip()):
+                elem.tail = indent