tellaro-query-language 0.1.0__py3-none-any.whl

tql/core_components/README.md

@@ -0,0 +1,92 @@
+# Core Components
+
+This package contains the modular components that implement TQL's core functionality.
+
+## Overview
+
+The core components package splits the TQL core functionality into specialized modules:
+
+### Components
+
+#### `opensearch_operations.py` - OpenSearch Operations
+Handles all OpenSearch-specific functionality:
+- Query conversion to OpenSearch DSL
+- Query execution against OpenSearch clusters
+- Mutator analysis and optimization
+- Phase 1/Phase 2 query splitting for optimal performance
+- Result post-processing
+
+**Key Methods:**
+- `to_opensearch()` - Convert TQL to OpenSearch query
+- `execute_opensearch()` - Execute query and return results
+- `analyze_opensearch_query()` - Analyze query optimization opportunities (internal use)
+
+#### `file_operations.py` - File I/O Operations
+Manages file loading and saving:
+- JSON file support with pretty printing
+- CSV file support (read-only)
+- Enrichment saving back to source files
+- Automatic file type detection
+
+**Key Methods:**
+- `load_file()` - Load data from JSON/CSV files
+- `save_enrichments_to_json()` - Save enriched data back to JSON
+
+#### `stats_operations.py` - Statistics Operations
+Implements statistical aggregations:
+- Aggregation functions (count, sum, avg, min, max, etc.)
+- Group-by operations
+- Combined filter and stats queries
+- Stats query analysis
+
+**Key Methods:**
+- `stats()` - Execute stats-only queries
+- `query_stats()` - Execute combined filter + stats queries
+- `analyze_stats_query()` - Analyze stats query for issues
+
+#### `validation_operations.py` - Query Validation
+Provides comprehensive query validation:
+- Syntax validation via parsing
+- Field name validation against mappings
+- Type compatibility checking
+- Performance issue detection
+- Query complexity analysis
+
+**Key Methods:**
+- `validate()` - Validate query syntax and fields
+- `check_type_compatibility()` - Verify operator/field type compatibility
+- `check_performance_issues()` - Identify potential performance problems
+
+## Usage
+
+These components are used internally by the main `TQL` class. They should not be imported directly:
+
+```python
+# Don't do this:
+from tql.core_components.opensearch_operations import OpenSearchOperations
+
+# Do this instead:
+from tql import TQL
+tql = TQL()
+results = tql.execute_opensearch("status = 'active'", index="logs")
+```
+
+## Architecture
+
+The core follows a modular architecture:
+
+```
+TQL (main class)
+    ├── OpenSearchOperations (OpenSearch integration)
+    ├── FileOperations (file I/O)
+    ├── StatsOperations (aggregations)
+    └── ValidationOperations (validation)
+```
+
+## Design Principles
+
+1. **Separation of Concerns**: Each component handles a specific domain
+2. **Dependency Injection**: Components receive dependencies via constructor
+3. **Stateless Operations**: Methods are mostly stateless for better testability
+4. **Error Propagation**: Components raise specific TQL exceptions
+5. **Type Safety**: Full type hints for better IDE support

tql/core_components/__init__.py

@@ -0,0 +1,20 @@
+"""Core components package for TQL.
+
+This package organizes TQL core functionality into logical modules:
+- opensearch_operations: OpenSearch query conversion and execution
+- file_operations: File loading and saving
+- stats_operations: Statistical aggregations
+- validation_operations: Query validation and type checking
+"""
+
+from .file_operations import FileOperations
+from .opensearch_operations import OpenSearchOperations
+from .stats_operations import StatsOperations
+from .validation_operations import ValidationOperations
+
+__all__ = [
+    "OpenSearchOperations",
+    "FileOperations",
+    "StatsOperations",
+    "ValidationOperations",
+]

tql/core_components/file_operations.py

@@ -0,0 +1,113 @@
+"""File operations for TQL.
+
+This module handles loading data from files and saving enrichments back to files.
+"""
+
+import csv
+import json
+import os
+from typing import Any, Dict, List
+
+from ..exceptions import TQLExecutionError
+
+
+class FileOperations:
+    """Handles file-based operations for TQL."""
+
+    def load_file(self, file_path: str) -> List[Dict[str, Any]]:  # noqa: C901
+        """Load data from a file (JSON or CSV).
+
+        Args:
+            file_path: Path to the file to load
+
+        Returns:
+            List of dictionaries representing the data
+
+        Raises:
+            TQLExecutionError: If file loading fails
+        """
+        if not os.path.exists(file_path):
+            raise TQLExecutionError(f"File not found: {file_path}")
+
+        _, ext = os.path.splitext(file_path.lower())
+
+        try:
+            if ext == ".json":
+                with open(file_path, "r", encoding="utf-8") as f:
+                    data = json.load(f)
+                    if isinstance(data, dict):
+                        # Single object, wrap in list
+                        return [data]
+                    elif isinstance(data, list):
+                        return data
+                    else:
+                        raise TQLExecutionError(f"Invalid JSON format in {file_path}")
+            elif ext == ".csv":
+                with open(file_path, "r", encoding="utf-8") as f:
+                    reader = csv.DictReader(f)
+                    return list(reader)
+            elif ext == ".jsonl":
+                # JSON Lines format - one JSON object per line
+                records = []
+                with open(file_path, "r", encoding="utf-8") as f:
+                    for line in f:
+                        line = line.strip()
+                        if line:  # Skip empty lines
+                            records.append(json.loads(line))
+                return records
+            else:
+                raise TQLExecutionError(f"Unsupported file format: {ext}")
+        except Exception as e:
+            raise TQLExecutionError(f"Error loading file {file_path}: {str(e)}")
+
+    def save_enrichments_to_json(self, file_path: str, records: List[Dict[str, Any]]) -> None:
+        """Save enriched records back to JSON file.
+
+        Args:
+            file_path: Path to save the file
+            records: List of records to save
+        """
+        with open(file_path, "w", encoding="utf-8") as f:
+            json.dump(records, f, indent=2, ensure_ascii=False)
+
+    def bulk_update_opensearch(
+        self, client: Any, index: str, records: List[Dict[str, Any]], id_field: str = "_id", batch_size: int = 100
+    ) -> Dict[str, int]:
+        """Bulk update records in OpenSearch.
+
+        Args:
+            client: OpenSearch client instance
+            index: Index name
+            records: Records to update
+            id_field: Field containing document ID
+            batch_size: Number of documents per bulk request
+
+        Returns:
+            Dictionary with update statistics
+        """
+        from opensearchpy.helpers import bulk as opensearch_bulk
+
+        updated = 0
+        failed = 0
+
+        # Process in batches
+        for i in range(0, len(records), batch_size):
+            batch = records[i : i + batch_size]
+            actions = []
+
+            for record in batch:
+                if id_field in record:
+                    action = {
+                        "_op_type": "update",
+                        "_index": index,
+                        "_id": record[id_field],
+                        "doc": {k: v for k, v in record.items() if k != id_field},
+                    }
+                    actions.append(action)
+
+            if actions:
+                success, failures = opensearch_bulk(client, actions, raise_on_error=False)
+                updated += success
+                failed += len(failures)
+
+        return {"updated": updated, "failed": failed}