moai-adk 0.35.1__py3-none-any.whl

moai_adk/templates/.claude/skills/moai-formats-data/modules/json-optimization.md

@@ -0,0 +1,374 @@
+# JSON/YAML Optimization Implementation
+
+> Module: High-performance JSON and YAML processing
+> Complexity: Advanced
+> Time: 20+ minutes
+> Dependencies: orjson, PyYAML, ijson, typing, dataclasses, functools
+
+## High-Performance JSON Handling
+
+```python
+import orjson # Ultra-fast JSON library
+import yaml
+from typing import Any, Dict, List
+from dataclasses import dataclass
+from functools import lru_cache
+
+class JSONOptimizer:
+ """Optimized JSON processing for high-performance applications."""
+
+ def __init__(self):
+ self._compression_cache = {}
+
+ def serialize_fast(self, obj: Any) -> bytes:
+ """Ultra-fast JSON serialization using orjson."""
+ return orjson.dumps(
+ obj,
+ option=orjson.OPT_SERIALIZE_NUMPY |
+ orjson.OPT_SERIALIZE_DATACLASS |
+ orjson.OPT_SERIALIZE_UUID |
+ orjson.OPT_NON_STR_KEYS
+ )
+
+ def deserialize_fast(self, data: bytes) -> Any:
+ """Ultra-fast JSON deserialization."""
+ return orjson.loads(data)
+
+ @lru_cache(maxsize=1024)
+ def compress_schema(self, schema: Dict) -> Dict:
+ """Cache and compress JSON schemas for repeated use."""
+ return self._optimize_schema(schema)
+
+ def _optimize_schema(self, schema: Dict) -> Dict:
+ """Remove redundant schema properties and optimize structure."""
+ optimized = {}
+
+ for key, value in schema.items():
+ if key == '$schema': # Remove schema URL
+ continue
+ elif key == 'description' and len(value) > 100: # Truncate long descriptions
+ optimized[key] = value[:100] + '...'
+ elif isinstance(value, dict):
+ optimized[key] = self._optimize_schema(value)
+ elif isinstance(value, list):
+ optimized[key] = [self._optimize_schema(item) if isinstance(item, dict) else item for item in value]
+ else:
+ optimized[key] = value
+
+ return optimized
+
+class YAMLOptimizer:
+ """Optimized YAML processing for configuration management."""
+
+ def __init__(self):
+ self.yaml_loader = yaml.CSafeLoader # Use C loader for performance
+ self.yaml_dumper = yaml.CSafeDumper
+
+ def load_fast(self, stream) -> Any:
+ """Fast YAML loading with optimized loader."""
+ return yaml.load(stream, Loader=self.yaml_loader)
+
+ def dump_fast(self, data: Any, stream=None) -> str:
+ """Fast YAML dumping with optimized dumper."""
+ return yaml.dump(
+ data,
+ stream=stream,
+ Dumper=self.yaml_dumper,
+ default_flow_style=False,
+ sort_keys=False,
+ allow_unicode=True
+ )
+
+ def merge_configs(self, *configs: Dict) -> Dict:
+ """Intelligently merge multiple YAML configurations."""
+ result = {}
+
+ for config in configs:
+ result = self._deep_merge(result, config)
+
+ return result
+
+ def _deep_merge(self, base: Dict, overlay: Dict) -> Dict:
+ """Deep merge dictionaries with conflict resolution."""
+ result = base.copy()
+
+ for key, value in overlay.items():
+ if key in result:
+ if isinstance(result[key], dict) and isinstance(value, dict):
+ result[key] = self._deep_merge(result[key], value)
+ elif isinstance(result[key], list) and isinstance(value, list):
+ result[key] = result[key] + value
+ else:
+ result[key] = value
+ else:
+ result[key] = value
+
+ return result
+
+# Streaming data processor for large datasets
+class StreamProcessor:
+ """Memory-efficient streaming processor for large data files."""
+
+ def __init__(self, chunk_size: int = 8192):
+ self.chunk_size = chunk_size
+
+ def process_json_stream(self, file_path: str, processor_func):
+ """Process large JSON files in streaming mode."""
+ import ijson
+
+ with open(file_path, 'rb') as file:
+ parser = ijson.parse(file)
+
+ current_object = {}
+ for prefix, event, value in parser:
+ if prefix.endswith('.item'):
+ if event == 'start_map':
+ current_object = {}
+ elif event == 'map_key':
+ self._current_key = value
+ elif event in ['string', 'number', 'boolean', 'null']:
+ current_object[self._current_key] = value
+ elif event == 'end_map':
+ processor_func(current_object)
+
+ def process_csv_stream(self, file_path: str, processor_func):
+ """Process large CSV files in streaming mode."""
+ import csv
+
+ with open(file_path, 'r', encoding='utf-8') as file:
+ reader = csv.DictReader(file)
+ for row in reader:
+ processor_func(row)
+
+ def aggregate_json_stream(self, file_path: str, aggregation_key: str) -> Dict:
+ """Aggregate streaming JSON data by key."""
+ aggregates = {}
+
+ def aggregate_processor(item):
+ key = item.get(aggregation_key, 'unknown')
+ if key not in aggregates:
+ aggregates[key] = {
+ 'count': 0,
+ 'items': []
+ }
+ aggregates[key]['count'] += 1
+ aggregates[key]['items'].append(item)
+
+ self.process_json_stream(file_path, aggregate_processor)
+ return aggregates
+```
+
+## Performance Optimization Techniques
+
+### Memory Management
+
+```python
+class MemoryEfficientProcessor:
+ """Memory-optimized data processing for large datasets."""
+
+ def __init__(self, max_memory_mb: int = 512):
+ self.max_memory_mb = max_memory_mb
+ self.current_memory = 0
+
+ def batch_process(self, data_generator, batch_size: int = 1000):
+ """Process data in batches to control memory usage."""
+ batch = []
+
+ for item in data_generator:
+ batch.append(item)
+
+ # Check memory usage and process batch
+ if len(batch) >= batch_size:
+ yield self._process_batch(batch)
+ batch.clear()
+
+ # Process remaining items
+ if batch:
+ yield self._process_batch(batch)
+
+ def _process_batch(self, batch: List[Any]) -> Any:
+ """Process a single batch of data."""
+ # Implement batch processing logic
+ pass
+
+ def _estimate_memory_usage(self, data: Any) -> int:
+ """Estimate memory usage of data structure."""
+ import sys
+ return sys.getsizeof(data)
+```
+
+### Caching Strategies
+
+```python
+from functools import wraps
+import hashlib
+import pickle
+import time
+
+class SmartCache:
+ """Intelligent caching with memory management and expiration."""
+
+ def __init__(self, max_size: int = 1000, ttl: int = 3600):
+ self.cache = {}
+ self.access_times = {}
+ self.max_size = max_size
+ self.ttl = ttl
+
+ def get_cache_key(self, func_name: str, args: tuple, kwargs: dict) -> str:
+ """Generate consistent cache key."""
+ key_data = f"{func_name}:{pickle.dumps(args)}:{pickle.dumps(kwargs)}"
+ return hashlib.md5(key_data.encode()).hexdigest()
+
+ def get(self, key: str) -> Any:
+ """Get value from cache with expiration check."""
+ if key in self.cache:
+ entry = self.cache[key]
+ if time.time() - entry['timestamp'] < self.ttl:
+ self.access_times[key] = time.time()
+ return entry['value']
+ else:
+ del self.cache[key]
+ del self.access_times[key]
+ return None
+
+ def set(self, key: str, value: Any):
+ """Set value in cache with LRU eviction."""
+ # Evict old entries if at capacity
+ while len(self.cache) >= self.max_size:
+ oldest_key = min(self.access_times.keys(),
+ key=lambda k: self.access_times[k])
+ del self.cache[oldest_key]
+ del self.access_times[oldest_key]
+
+ self.cache[key] = {
+ 'value': value,
+ 'timestamp': time.time()
+ }
+ self.access_times[key] = time.time()
+
+def cached(ttl: int = 3600):
+ """Decorator for smart function caching."""
+ cache = SmartCache(ttl=ttl)
+
+ def decorator(func):
+ @wraps(func)
+ def wrapper(*args, kwargs):
+ cache_key = cache.get_cache_key(func.__name__, args, kwargs)
+
+ # Try to get from cache
+ cached_result = cache.get(cache_key)
+ if cached_result is not None:
+ return cached_result
+
+ # Execute and cache result
+ result = func(*args, kwargs)
+ cache.set(cache_key, result)
+ return result
+
+ return wrapper
+ return decorator
+```
+
+### Format-Specific Optimizations
+
+```python
+class FormatSpecificOptimizer:
+ """Optimizations for specific data formats."""
+
+ def optimize_for_json(self, data: Dict) -> Dict:
+ """Optimize data structure for JSON serialization."""
+ optimized = {}
+
+ for key, value in data.items():
+ # Convert datetime objects to ISO strings
+ if hasattr(value, 'isoformat'):
+ optimized[key] = value.isoformat()
+ # Convert sets to lists for JSON compatibility
+ elif isinstance(value, set):
+ optimized[key] = list(value)
+ # Handle None values
+ elif value is None:
+ # Skip None values for compactness (optional)
+ continue
+ else:
+ optimized[key] = value
+
+ return optimized
+
+ def optimize_for_yaml(self, data: Dict) -> Dict:
+ """Optimize data structure for YAML serialization."""
+ optimized = {}
+
+ for key, value in data.items():
+ # Preserve complex objects for YAML's better type support
+ if hasattr(value, '__dict__'):
+ optimized[key] = value
+ # Use multi-line strings for long text
+ elif isinstance(value, str) and len(value) > 100:
+ optimized[key] = self._format_multiline_string(value)
+ else:
+ optimized[key] = value
+
+ return optimized
+
+ def _format_multiline_string(self, text: str) -> str:
+ """Format long strings as YAML multi-line."""
+ # Simple implementation - could be enhanced with smarter line breaks
+ lines = text.split('\n')
+ if len(lines) == 1:
+ return text
+ return '|\n' + '\n'.join(' ' + line for line in lines)
+
+# Performance monitoring
+class PerformanceMonitor:
+ """Monitor and optimize data processing performance."""
+
+ def __init__(self):
+ self.metrics = {}
+
+ def time_operation(self, operation_name: str):
+ """Decorator to time operations."""
+ def decorator(func):
+ @wraps(func)
+ def wrapper(*args, kwargs):
+ start_time = time.time()
+ result = func(*args, kwargs)
+ duration = time.time() - start_time
+
+ # Record metrics
+ if operation_name not in self.metrics:
+ self.metrics[operation_name] = []
+ self.metrics[operation_name].append(duration)
+
+ return result
+ return wrapper
+ return decorator
+
+ def get_stats(self, operation_name: str) -> Dict:
+ """Get performance statistics for an operation."""
+ if operation_name not in self.metrics:
+ return {}
+
+ durations = self.metrics[operation_name]
+ return {
+ 'count': len(durations),
+ 'total_time': sum(durations),
+ 'avg_time': sum(durations) / len(durations),
+ 'min_time': min(durations),
+ 'max_time': max(durations)
+ }
+```
+
+## Best Practices
+
+1. Use orjson for JSON: 2-5x faster than standard json module
+2. Stream large files: Use ijson for memory-efficient processing
+3. Cache intelligently: Implement smart caching with expiration
+4. Batch processing: Process data in manageable chunks
+5. Profile regularly: Monitor performance bottlenecks
+6. Choose right format: YAML for config, JSON for data exchange, TOON for LLM communication
+
+---
+
+Module: `modules/json-optimization.md`
+Related: [TOON Encoding](./toon-encoding.md) | [Data Validation](./data-validation.md)

moai_adk/templates/.claude/skills/moai-formats-data/modules/toon-encoding.md

@@ -0,0 +1,308 @@
+# TOON Encoding Implementation
+
+> Module: Core TOON (Token-Optimized Object Notation) implementation
+> Complexity: Advanced
+> Time: 15+ minutes
+> Dependencies: Python 3.8+, typing, datetime, json
+
+## Core TOON Implementation
+
+```python
+from typing import Dict, List, Any, Union
+import json
+from datetime import datetime
+
+class TOONEncoder:
+ """Token-Optimized Object Notation encoder for efficient LLM communication."""
+
+ def __init__(self):
+ self.type_markers = {
+ 'string': '',
+ 'number': '#',
+ 'boolean': '!',
+ 'null': '~',
+ 'timestamp': '@'
+ }
+
+ def encode(self, data: Any) -> str:
+ """Encode Python data structure to TOON format."""
+ return self._encode_value(data)
+
+ def _encode_value(self, value: Any) -> str:
+ """Encode individual values with type optimization."""
+
+ if value is None:
+ return '~'
+
+ elif isinstance(value, bool):
+ return f'!{str(value)[0]}' # !t or !f
+
+ elif isinstance(value, (int, float)):
+ return f'#{value}'
+
+ elif isinstance(value, str):
+ return self._escape_string(value)
+
+ elif isinstance(value, datetime):
+ return f'@{value.isoformat()}'
+
+ elif isinstance(value, dict):
+ if not value:
+ return '{}'
+ items = []
+ for k, v in value.items():
+ encoded_key = self._escape_string(str(k))
+ encoded_value = self._encode_value(v)
+ items.append(f'{encoded_key}:{encoded_value}')
+ return '{' + ','.join(items) + '}'
+
+ elif isinstance(value, list):
+ if not value:
+ return '[]'
+ encoded_items = [self._encode_value(item) for item in value]
+ return '[' + '|'.join(encoded_items) + ']'
+
+ else:
+ # Fallback to JSON for complex objects
+ json_str = json.dumps(value, default=str)
+ return f'${json_str}'
+
+ def _escape_string(self, s: str) -> str:
+ """Escape special characters in strings."""
+ # Replace problematic characters
+ s = s.replace('\\', '\\\\')
+ s = s.replace(':', '\\:')
+ s = s.replace('|', '\\|')
+ s = s.replace(',', '\\,')
+ s = s.replace('{', '\\{')
+ s = s.replace('}', '\\}')
+ s = s.replace('[', '\\[')
+ s = s.replace(']', '\\]')
+ s = s.replace('~', '\\~')
+ s = s.replace('#', '\\#')
+ s = s.replace('!', '\\!')
+ s = s.replace('@', '\\@')
+ s = s.replace('$', '\\$')
+
+ return s
+
+ def decode(self, toon_str: str) -> Any:
+ """Decode TOON format back to Python data structure."""
+ return self._parse_value(toon_str.strip())
+
+ def _parse_value(self, s: str) -> Any:
+ """Parse TOON value back to Python type."""
+ s = s.strip()
+
+ if not s:
+ return None
+
+ # Null value
+ if s == '~':
+ return None
+
+ # Boolean values
+ if s.startswith('!'):
+ return s[1:] == 't'
+
+ # Numbers
+ if s.startswith('#'):
+ num_str = s[1:]
+ if '.' in num_str:
+ return float(num_str)
+ return int(num_str)
+
+ # Timestamps
+ if s.startswith('@'):
+ try:
+ return datetime.fromisoformat(s[1:])
+ except ValueError:
+ return s[1:] # Return as string if parsing fails
+
+ # JSON fallback
+ if s.startswith('$'):
+ return json.loads(s[1:])
+
+ # Arrays
+ if s.startswith('[') and s.endswith(']'):
+ content = s[1:-1]
+ if not content:
+ return []
+ items = self._split_array_items(content)
+ return [self._parse_value(item) for item in items]
+
+ # Objects
+ if s.startswith('{') and s.endswith('}'):
+ content = s[1:-1]
+ if not content:
+ return {}
+ pairs = self._split_object_pairs(content)
+ result = {}
+ for pair in pairs:
+ if ':' in pair:
+ key, value = pair.split(':', 1)
+ result[self._unescape_string(key)] = self._parse_value(value)
+ return result
+
+ # String (default)
+ return self._unescape_string(s)
+
+ def _split_array_items(self, content: str) -> List[str]:
+ """Split array items handling escaped separators."""
+ items = []
+ current = []
+ escape = False
+
+ for char in content:
+ if escape:
+ current.append(char)
+ escape = False
+ elif char == '\\':
+ escape = True
+ elif char == '|':
+ items.append(''.join(current))
+ current = []
+ else:
+ current.append(char)
+
+ if current:
+ items.append(''.join(current))
+
+ return items
+
+ def _split_object_pairs(self, content: str) -> List[str]:
+ """Split object pairs handling escaped separators."""
+ pairs = []
+ current = []
+ escape = False
+ depth = 0
+
+ for char in content:
+ if escape:
+ current.append(char)
+ escape = False
+ elif char == '\\':
+ escape = True
+ elif char in '{[':
+ depth += 1
+ current.append(char)
+ elif char in '}]':
+ depth -= 1
+ current.append(char)
+ elif char == ',' and depth == 0:
+ pairs.append(''.join(current))
+ current = []
+ else:
+ current.append(char)
+
+ if current:
+ pairs.append(''.join(current))
+
+ return pairs
+
+ def _unescape_string(self, s: str) -> str:
+ """Unescape escaped characters in strings."""
+ return s.replace('\\:', ':').replace('\\|', '|').replace('\\,', ',') \
+ .replace('\\{', '{').replace('\\}', '}').replace('\\[', '[') \
+ .replace('\\]', ']').replace('\\~', '~').replace('\\#', '#') \
+ .replace('\\!', '!').replace('\\@', '@').replace('\\$', '$') \
+ .replace('\\\\', '\\')
+
+# Usage example and performance comparison
+def demonstrate_toon_optimization():
+ data = {
+ "user": {
+ "id": 12345,
+ "name": "John Doe",
+ "email": "john.doe@example.com",
+ "active": True,
+ "created_at": datetime.now()
+ },
+ "permissions": ["read", "write", "admin"],
+ "settings": {
+ "theme": "dark",
+ "notifications": True
+ }
+ }
+
+ encoder = TOONEncoder()
+
+ # JSON encoding
+ json_str = json.dumps(data, default=str)
+ json_tokens = len(json_str.split())
+
+ # TOON encoding
+ toon_str = encoder.encode(data)
+ toon_tokens = len(toon_str.split())
+
+ # Performance comparison
+ reduction = (1 - toon_tokens / json_tokens) * 100
+
+ return {
+ "json_size": len(json_str),
+ "toon_size": len(toon_str),
+ "json_tokens": json_tokens,
+ "toon_tokens": toon_tokens,
+ "token_reduction": reduction,
+ "json_str": json_str,
+ "toon_str": toon_str
+ }
+```
+
+## TOON Format Specification
+
+### Type Markers
+- string: No marker (default)
+- number: # prefix
+- boolean: ! prefix (!t, !f)
+- null: ~
+- timestamp: @ prefix (ISO 8601)
+- json-fallback: $ prefix (JSON-encoded)
+
+### Structure Rules
+- Objects: `{key1:value1,key2:value2}`
+- Arrays: `[item1|item2|item3]`
+- Escaping: Backslash `\` for special characters
+- Separators: `,` for objects, `|` for arrays
+
+### Performance Characteristics
+- Token Reduction: 40-60% vs JSON for typical data
+- Parsing Speed: 2-3x faster than JSON for simple structures
+- Size Reduction: 30-50% smaller than JSON for most use cases
+- Compatibility: Lossless round-trip encoding/decoding
+
+### Advanced Usage Patterns
+
+```python
+# Batch processing
+def encode_batch(data_list: List[Dict]) -> List[str]:
+ encoder = TOONEncoder()
+ return [encoder.encode(data) for data in data_list]
+
+# Streaming TOON processing
+def stream_toon_file(file_path: str):
+ with open(file_path, 'r') as f:
+ encoder = TOONEncoder()
+ for line in f:
+ data = json.loads(line.strip())
+ toon_data = encoder.encode(data)
+ yield toon_data
+
+# Integration with LLM APIs
+def prepare_for_llm(data: Any, max_tokens: int = 1000) -> str:
+ encoder = TOONEncoder()
+ toon_data = encoder.encode(data)
+
+ # Further compression if needed
+ while len(toon_data.split()) > max_tokens:
+ # Implement selective data reduction
+ data = reduce_data_complexity(data)
+ toon_data = encoder.encode(data)
+
+ return toon_data
+```
+
+---
+
+Module: `modules/toon-encoding.md`
+Related: [JSON Optimization](./json-optimization.md) | [Data Validation](./data-validation.md)