contextsynapse 1.0.0__py3-none-any.whl

contextsynapse/buffer/file_buffer.py

@@ -0,0 +1,229 @@
+"""
+File-Based Buffer Manager
+
+Simple file-based buffer for single-node deployments.
+Uses JSON Lines format for easy debugging.
+"""
+
+import asyncio
+import json
+import time
+from pathlib import Path
+from typing import Dict, Any, Optional, List, Callable, Awaitable
+from .base import BufferManager, BufferConfig
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class FileBufferManager(BufferManager):
+    """
+    File-based buffer manager.
+    
+    Stores records in a JSON Lines file (one JSON object per line).
+    Perfect for:
+    - Single-node deployments
+    - Simple persistence
+    - Easy debugging (human-readable format)
+    
+    Not suitable for:
+    - High-concurrency (file locking overhead)
+    - Very high throughput
+    """
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self._file_path: Optional[Path] = None
+        self._file_handle = None
+    
+    async def initialize(self) -> None:
+        """Initialize file buffer."""
+        async with self._lock:
+            # Determine file path
+            if self.config.path:
+                self._file_path = Path(self.config.path)
+            else:
+                # Default path
+                buffer_dir = Path("contextcore_data") / "buffers"
+                if self.config.namespace:
+                    buffer_dir = buffer_dir / self.config.namespace
+                buffer_dir.mkdir(parents=True, exist_ok=True)
+                self._file_path = buffer_dir / "buffer.jsonl"
+            
+            # Ensure directory exists
+            self._file_path.parent.mkdir(parents=True, exist_ok=True)
+            
+            # Open file in append mode
+            self._file_handle = open(self._file_path, 'a', encoding='utf-8')
+            
+            self._initialized = True
+            logger.info(f"[EMOJI] FileBufferManager initialized: {self._file_path}")
+    
+    async def add_record(self, data: Dict[str, Any]) -> None:
+        """Add record to file buffer."""
+        async with self._lock:
+            if not self._file_handle:
+                await self.initialize()
+            
+            # Add timestamp
+            record = {
+                'timestamp': time.time(),
+                'data': data
+            }
+            
+            # Write as JSON line
+            json_line = json.dumps(record, default=str) + '\n'
+            self._file_handle.write(json_line)
+            self._file_handle.flush()  # Ensure written to disk
+    
+    async def flush_to_db(
+        self,
+        batch_size: Optional[int] = None,
+        writer_fn: Optional[Callable[[List[Dict[str, Any]]], Awaitable[None]]] = None
+    ) -> int:
+        """Flush records from file to database."""
+        if not writer_fn:
+            logger.warning("No writer function provided, skipping flush")
+            return 0
+        
+        async with self._lock:
+            if not self._file_path or not self._file_path.exists():
+                return 0
+            
+            # Close current file handle
+            if self._file_handle:
+                self._file_handle.close()
+                self._file_handle = None
+            
+            # Read all records
+            records = []
+            with open(self._file_path, 'r', encoding='utf-8') as f:
+                for line in f:
+                    if line.strip():
+                        try:
+                            record = json.loads(line)
+                            records.append(record['data'])
+                        except json.JSONDecodeError as e:
+                            logger.warning(f"Invalid JSON line: {e}")
+            
+            if not records:
+                # Reopen file for appending
+                self._file_handle = open(self._file_path, 'a', encoding='utf-8')
+                return 0
+            
+            # Process in batches
+            batch_size = batch_size or self.config.batch_size
+            flushed_count = 0
+            failed_records = []
+            
+            for i in range(0, len(records), batch_size):
+                batch = records[i:i + batch_size]
+                try:
+                    await writer_fn(batch)
+                    flushed_count += len(batch)
+                    logger.debug(f"Flushed batch of {len(batch)} records")
+                except Exception as e:
+                    logger.error(f"Error writing batch: {e}")
+                    failed_records.extend(batch)
+            
+            # Write failed records back to file
+            if failed_records:
+                with open(self._file_path, 'w', encoding='utf-8') as f:
+                    for record in failed_records:
+                        json_line = json.dumps({'timestamp': time.time(), 'data': record}, default=str) + '\n'
+                        f.write(json_line)
+            else:
+                # Clear file if all records flushed successfully
+                self._file_path.write_text('')
+            
+            # Reopen file for appending
+            self._file_handle = open(self._file_path, 'a', encoding='utf-8')
+            
+            if flushed_count > 0:
+                logger.info(f"[EMOJI] Flushed {flushed_count} records from file buffer")
+            
+            return flushed_count
+    
+    async def get_pending_count(self) -> int:
+        """Get number of pending records in file."""
+        async with self._lock:
+            if not self._file_path or not self._file_path.exists():
+                return 0
+            
+            # Count lines in file
+            try:
+                with open(self._file_path, 'r', encoding='utf-8') as f:
+                    return sum(1 for line in f if line.strip())
+            except Exception as e:
+                logger.error(f"Error counting records: {e}")
+                return 0
+    
+    async def cleanup(self, older_than_seconds: Optional[float] = None) -> int:
+        """Cleanup old records from file."""
+        async with self._lock:
+            if not self._file_path or not self._file_path.exists():
+                return 0
+            
+            # Close current handle
+            if self._file_handle:
+                self._file_handle.close()
+                self._file_handle = None
+            
+            # Read all records
+            records = []
+            with open(self._file_path, 'r', encoding='utf-8') as f:
+                for line in f:
+                    if line.strip():
+                        try:
+                            record = json.loads(line)
+                            records.append(record)
+                        except json.JSONDecodeError:
+                            continue
+            
+            if not records:
+                self._file_handle = open(self._file_path, 'a', encoding='utf-8')
+                return 0
+            
+            # Filter records
+            current_time = time.time()
+            kept_records = []
+            removed_count = 0
+            
+            for record in records:
+                timestamp = record.get('timestamp', 0)
+                age = current_time - timestamp
+                
+                if older_than_seconds is None or age <= older_than_seconds:
+                    kept_records.append(record)
+                else:
+                    removed_count += 1
+            
+            # Write kept records back
+            with open(self._file_path, 'w', encoding='utf-8') as f:
+                for record in kept_records:
+                    json_line = json.dumps(record, default=str) + '\n'
+                    f.write(json_line)
+            
+            # Reopen for appending
+            self._file_handle = open(self._file_path, 'a', encoding='utf-8')
+            
+            if removed_count > 0:
+                logger.info(f"[EMOJI] Cleaned up {removed_count} old records from file buffer")
+            
+            return removed_count
+    
+    async def close(self) -> None:
+        """Close buffer manager."""
+        async with self._lock:
+            if self._file_handle:
+                self._file_handle.close()
+                self._file_handle = None
+            self._initialized = False
+            logger.debug("FileBufferManager closed")
+
+
+
+
+
+
+

contextsynapse/buffer/lmdb_buffer.py

@@ -0,0 +1,286 @@
+"""
+LMDB Buffer Manager
+
+LMDB (Lightning Memory-Mapped Database) buffer for high-performance scenarios.
+Perfect for high-throughput, low-latency requirements.
+"""
+
+import asyncio
+import json
+import time
+from pathlib import Path
+from typing import Dict, Any, Optional, List, Callable, Awaitable
+from .base import BufferManager, BufferConfig
+import logging
+
+logger = logging.getLogger(__name__)
+
+try:
+    import lmdb
+    LMDB_AVAILABLE = True
+except ImportError:
+    LMDB_AVAILABLE = False
+    logger.warning("lmdb package not available. Install with: pip install lmdb")
+
+
+class LMDBBufferManager(BufferManager):
+    """
+    LMDB-based buffer manager.
+    
+    Perfect for:
+    - High-throughput scenarios
+    - Low-latency requirements
+    - Memory-mapped performance
+    - Single-writer, multiple-reader scenarios
+    
+    Requires:
+    - LMDB Python package
+    - LMDB C library
+    """
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self._db_path: Optional[Path] = None
+        self._env = None
+        self._db = None
+        self._next_id = 0
+    
+    async def initialize(self) -> None:
+        """Initialize LMDB environment."""
+        if not LMDB_AVAILABLE:
+            raise RuntimeError("LMDB package not available. Install with: pip install lmdb")
+        
+        async with self._lock:
+            # Determine database path
+            if self.config.path:
+                self._db_path = Path(self.config.path)
+            else:
+                buffer_dir = Path("contextcore_data") / "buffers"
+                if self.config.namespace:
+                    buffer_dir = buffer_dir / self.config.namespace
+                buffer_dir.mkdir(parents=True, exist_ok=True)
+                self._db_path = buffer_dir / "lmdb"
+            
+            # Ensure directory exists
+            self._db_path.mkdir(parents=True, exist_ok=True)
+            
+            # Get LMDB config
+            map_size = self.config.backend_config.get('map_size', 1024 * 1024 * 1024)  # 1GB default
+            
+            # Open LMDB environment (sync API, run in executor)
+            loop = asyncio.get_event_loop()
+            self._env = await loop.run_in_executor(
+                None,
+                lambda: lmdb.open(
+                    str(self._db_path),
+                    map_size=map_size,
+                    max_dbs=1
+                )
+            )
+            
+            # Open database
+            self._db = await loop.run_in_executor(
+                None,
+                lambda: self._env.open_db(b'buffer_records')
+            )
+            
+            # Get next ID
+            with self._env.begin(db=self._db, write=False) as txn:
+                cursor = txn.cursor()
+                if cursor.last():
+                    last_key = cursor.key()
+                    self._next_id = int.from_bytes(last_key, 'big') + 1
+                else:
+                    self._next_id = 1
+            
+            self._initialized = True
+            logger.info(f"[EMOJI] LMDBBufferManager initialized: {self._db_path}")
+    
+    async def add_record(self, data: Dict[str, Any]) -> None:
+        """Add record to LMDB buffer."""
+        if not self._env:
+            await self.initialize()
+        
+        # Serialize record
+        record = {
+            'timestamp': time.time(),
+            'data': data
+        }
+        record_json = json.dumps(record, default=str).encode('utf-8')
+        
+        # Get ID and increment
+        record_id = self._next_id
+        self._next_id += 1
+        
+        # Write to LMDB (sync API, run in executor)
+        loop = asyncio.get_event_loop()
+        await loop.run_in_executor(
+            None,
+            lambda: self._write_record(record_id, record_json)
+        )
+    
+    def _write_record(self, record_id: int, record_json: bytes):
+        """Sync write to LMDB."""
+        key = record_id.to_bytes(8, 'big')
+        with self._env.begin(db=self._db, write=True) as txn:
+            txn.put(key, record_json)
+    
+    async def flush_to_db(
+        self,
+        batch_size: Optional[int] = None,
+        writer_fn: Optional[Callable[[List[Dict[str, Any]]], Awaitable[None]]] = None
+    ) -> int:
+        """Flush records from LMDB to database."""
+        if not writer_fn:
+            logger.warning("No writer function provided, skipping flush")
+            return 0
+        
+        if not self._env:
+            return 0
+        
+        batch_size = batch_size or self.config.batch_size
+        flushed_count = 0
+        loop = asyncio.get_event_loop()
+        
+        while True:
+            # Read batch (sync API, run in executor)
+            batch_data = await loop.run_in_executor(
+                None,
+                lambda: self._read_batch(batch_size)
+            )
+            
+            if not batch_data:
+                break
+            
+            batch, keys = batch_data
+            
+            # Write batch
+            try:
+                await writer_fn(batch)
+                flushed_count += len(batch)
+                
+                # Delete flushed records (sync API, run in executor)
+                await loop.run_in_executor(
+                    None,
+                    lambda: self._delete_records(keys)
+                )
+                
+                logger.debug(f"Flushed batch of {len(batch)} records from LMDB")
+            except Exception as e:
+                logger.error(f"Error writing batch: {e}")
+                raise
+        
+        if flushed_count > 0:
+            logger.info(f"[EMOJI] Flushed {flushed_count} records from LMDB buffer")
+        
+        return flushed_count
+    
+    def _read_batch(self, batch_size: int):
+        """Sync read batch from LMDB."""
+        batch = []
+        keys = []
+        
+        with self._env.begin(db=self._db, write=False) as txn:
+            cursor = txn.cursor()
+            for key, value in cursor:
+                if len(batch) >= batch_size:
+                    break
+                
+                try:
+                    record = json.loads(value.decode('utf-8'))
+                    batch.append(record['data'])
+                    keys.append(key)
+                except (json.JSONDecodeError, KeyError) as e:
+                    logger.warning(f"Invalid record: {e}")
+                    keys.append(key)  # Mark for deletion
+                    continue
+        
+        return (batch, keys) if batch else None
+    
+    def _delete_records(self, keys: List[bytes]):
+        """Sync delete records from LMDB."""
+        with self._env.begin(db=self._db, write=True) as txn:
+            for key in keys:
+                txn.delete(key)
+    
+    async def get_pending_count(self) -> int:
+        """Get number of pending records."""
+        if not self._env:
+            return 0
+        
+        loop = asyncio.get_event_loop()
+        count = await loop.run_in_executor(
+            None,
+            lambda: self._count_records()
+        )
+        return count
+    
+    def _count_records(self) -> int:
+        """Sync count records in LMDB."""
+        with self._env.begin(db=self._db, write=False) as txn:
+            return txn.stat()['entries']
+    
+    async def cleanup(self, older_than_seconds: Optional[float] = None) -> int:
+        """Cleanup old records."""
+        if not self._env:
+            return 0
+        
+        current_time = time.time()
+        loop = asyncio.get_event_loop()
+        removed_count = await loop.run_in_executor(
+            None,
+            lambda: self._cleanup_records(current_time, older_than_seconds)
+        )
+        
+        if removed_count > 0:
+            logger.info(f"[EMOJI] Cleaned up {removed_count} old records from LMDB buffer")
+        
+        return removed_count
+    
+    def _cleanup_records(self, current_time: float, older_than_seconds: Optional[float]) -> int:
+        """Sync cleanup records."""
+        removed_count = 0
+        keys_to_delete = []
+        
+        with self._env.begin(db=self._db, write=False) as txn:
+            cursor = txn.cursor()
+            for key, value in cursor:
+                try:
+                    record = json.loads(value.decode('utf-8'))
+                    timestamp = record.get('timestamp', 0)
+                    age = current_time - timestamp
+                    
+                    if older_than_seconds is None or age > older_than_seconds:
+                        keys_to_delete.append(key)
+                except (json.JSONDecodeError, KeyError):
+                    keys_to_delete.append(key)  # Delete invalid records
+        
+        # Delete records
+        if keys_to_delete:
+            with self._env.begin(db=self._db, write=True) as txn:
+                for key in keys_to_delete:
+                    txn.delete(key)
+                    removed_count += 1
+        
+        return removed_count
+    
+    async def close(self) -> None:
+        """Close LMDB environment."""
+        async with self._lock:
+            if self._env:
+                loop = asyncio.get_event_loop()
+                await loop.run_in_executor(
+                    None,
+                    lambda: self._env.close()
+                )
+                self._env = None
+                self._db = None
+            self._initialized = False
+            logger.debug("LMDBBufferManager closed")
+
+
+
+
+
+
+

contextsynapse/buffer/local_buffer.py

@@ -0,0 +1,154 @@
+"""
+Local In-Memory Buffer Manager
+
+Simple in-memory buffer for development and testing.
+Not suitable for production (data lost on restart).
+"""
+
+import asyncio
+import time
+from typing import Dict, Any, Optional, List, Callable, Awaitable
+from .base import BufferManager, BufferConfig
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class LocalBufferManager(BufferManager):
+    """
+    In-memory buffer manager.
+    
+    Perfect for:
+    - Development and testing
+    - Single-process applications
+    - Low-volume workloads
+    
+    Not suitable for:
+    - Production clusters
+    - High-availability requirements
+    - Data persistence across restarts
+    """
+    
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config)
+        self._records: List[Dict[str, Any]] = []
+        self._timestamps: List[float] = []
+    
+    async def initialize(self) -> None:
+        """Initialize local buffer (no-op for in-memory)."""
+        async with self._lock:
+            self._initialized = True
+            logger.info("[EMOJI] LocalBufferManager initialized (in-memory)")
+    
+    async def add_record(self, data: Dict[str, Any]) -> None:
+        """Add record to in-memory buffer with threshold check."""
+        async with self._lock:
+            self._records.append(data)
+            self._timestamps.append(time.time())
+            
+            current_size = len(self._records)
+            max_size = self.config.max_size
+            threshold = int(max_size * self.config.flush_threshold) if self.config.threshold_flush_enabled else max_size
+            
+            # STRATEGY: Check threshold (70% by default) and log if reached
+            if self.config.threshold_flush_enabled and current_size >= threshold:
+                logger.debug(f"Buffer threshold reached: {current_size}/{max_size} ({current_size/max_size*100:.1f}%) - will flush on next cycle")
+                # Note: Don't flush here directly in async context, let auto-flush handle it
+                # But the threshold check in add_node will trigger flush
+            
+            # Auto-flush if buffer is full (100%)
+            if current_size >= max_size:
+                logger.debug(f"Buffer size ({current_size}) >= max_size ({max_size}), triggering flush")
+                # Note: Don't flush here directly, let auto-flush handle it
+    
+    async def flush_to_db(
+        self,
+        batch_size: Optional[int] = None,
+        writer_fn: Optional[Callable[[List[Dict[str, Any]]], Awaitable[None]]] = None
+    ) -> int:
+        """Flush records to database."""
+        async with self._lock:
+            if not self._records:
+                return 0
+            
+            batch_size = batch_size or self.config.batch_size
+            flushed_count = 0
+            
+            # Process in batches
+            while self._records:
+                batch = self._records[:batch_size]
+                batch_timestamps = self._timestamps[:batch_size]
+                
+                # Remove from buffer
+                self._records = self._records[batch_size:]
+                self._timestamps = self._timestamps[batch_size:]
+                
+                # Write batch
+                if writer_fn:
+                    try:
+                        await writer_fn(batch)
+                        flushed_count += len(batch)
+                        logger.debug(f"Flushed batch of {len(batch)} records")
+                    except Exception as e:
+                        logger.error(f"Error writing batch: {e}")
+                        # Put records back on error
+                        self._records = batch + self._records
+                        self._timestamps = batch_timestamps + self._timestamps
+                        raise
+                else:
+                    # No writer function, just remove records
+                    flushed_count += len(batch)
+                    logger.debug(f"Removed {len(batch)} records (no writer function)")
+            
+            if flushed_count > 0:
+                logger.info(f"[EMOJI] Flushed {flushed_count} records from local buffer")
+            
+            return flushed_count
+    
+    async def get_pending_count(self) -> int:
+        """Get number of pending records."""
+        async with self._lock:
+            return len(self._records)
+    
+    async def cleanup(self, older_than_seconds: Optional[float] = None) -> int:
+        """Cleanup old records."""
+        async with self._lock:
+            if older_than_seconds is None:
+                # Remove all
+                count = len(self._records)
+                self._records.clear()
+                self._timestamps.clear()
+                return count
+            
+            # Remove records older than threshold
+            current_time = time.time()
+            kept_records = []
+            kept_timestamps = []
+            removed_count = 0
+            
+            for record, timestamp in zip(self._records, self._timestamps):
+                age = current_time - timestamp
+                if age <= older_than_seconds:
+                    kept_records.append(record)
+                    kept_timestamps.append(timestamp)
+                else:
+                    removed_count += 1
+            
+            self._records = kept_records
+            self._timestamps = kept_timestamps
+            
+            if removed_count > 0:
+                logger.info(f"[EMOJI] Cleaned up {removed_count} old records from local buffer")
+            
+            return removed_count
+    
+    async def close(self) -> None:
+        """Close buffer manager."""
+        async with self._lock:
+            self._records.clear()
+            self._timestamps.clear()
+            self._initialized = False
+            logger.debug("LocalBufferManager closed")
+
+
+