isage-middleware 0.2.4.3__cp311-none-any.whl

sage/middleware/components/sage_sias/continual_learner.py

@@ -0,0 +1,184 @@
+"""
+Online Continual Learning with Experience Replay
+
+Implements an experience replay buffer for online/incremental training that
+prevents catastrophic forgetting. The buffer is managed using coreset selection
+to retain the most valuable samples.
+
+This is a core component of SIAS (Streaming Importance-Aware Agent System).
+"""
+
+from __future__ import annotations
+
+import random
+from typing import Iterable, Optional, Sequence
+
+from .coreset_selector import CoresetSelector, SampleT, SelectionSummary
+
+
+class OnlineContinualLearner:
+    """
+    Maintain a replay buffer for online continual learning.
+
+    Implements experience replay to prevent catastrophic forgetting during
+    incremental/online training. The buffer is managed using coreset selection
+    to keep the most valuable samples.
+
+    Attributes:
+        buffer_size: Maximum number of samples to keep in buffer
+        replay_ratio: Ratio of replay samples to add per batch (e.g., 0.25 = 25%)
+        selector: CoresetSelector for buffer management
+
+    Example:
+        >>> learner = OnlineContinualLearner(buffer_size=2048, replay_ratio=0.25)
+        >>> for new_batch in data_stream:
+        ...     training_batch = learner.update_buffer(new_batch)
+        ...     train_step(training_batch)
+    """
+
+    def __init__(
+        self,
+        buffer_size: int = 2048,
+        replay_ratio: float = 0.3,
+        selector: Optional[CoresetSelector] = None,
+        random_seed: int = 17,
+    ) -> None:
+        """
+        Initialize OnlineContinualLearner.
+
+        Args:
+            buffer_size: Maximum samples to keep in replay buffer
+            replay_ratio: Fraction of batch size to sample from buffer
+            selector: CoresetSelector for buffer management (default: hybrid)
+            random_seed: Random seed for reproducibility
+        """
+        self.buffer_size = buffer_size
+        self.replay_ratio = replay_ratio
+        self.selector = selector or CoresetSelector(strategy="hybrid")
+        self._buffer: list[SampleT] = []
+        self._metrics: dict[str, float] = {}
+        self._rng = random.Random(random_seed)
+
+    @property
+    def buffer(self) -> list[SampleT]:
+        """Access the current buffer (read-only view)."""
+        return list(self._buffer)
+
+    @property
+    def buffer_len(self) -> int:
+        """Current number of samples in buffer."""
+        return len(self._buffer)
+
+    def update_buffer(
+        self,
+        new_samples: Sequence[SampleT],
+        metrics: Optional[dict[str, float]] = None,
+    ) -> list[SampleT]:
+        """
+        Update buffer with new samples and return training batch.
+
+        This method:
+        1. Adds new samples to the buffer
+        2. If buffer exceeds size limit, uses coreset selection to prune
+        3. Returns new samples + replay samples for training
+
+        Args:
+            new_samples: New samples to add to buffer
+            metrics: Optional metrics dict mapping sample_id to importance score
+
+        Returns:
+            Training batch combining new samples with replay samples
+        """
+        if not new_samples:
+            return list(self._buffer)
+
+        if metrics:
+            self._metrics.update(metrics)
+
+        # Combine buffer with new samples
+        combined = list(self._buffer) + list(new_samples)
+
+        # Prune if over capacity
+        if len(combined) > self.buffer_size:
+            combined = self.selector.select(
+                combined,
+                target_size=self.buffer_size,
+                metrics=self._metrics,
+            )
+            # Clean up metrics for removed samples
+            combined_ids = {self._get_sample_id(sample) for sample in combined}
+            self._metrics = {k: v for k, v in self._metrics.items() if k in combined_ids}
+
+        self._buffer = combined
+        return self._assemble_training_batch(new_samples)
+
+    def _assemble_training_batch(
+        self,
+        new_samples: Sequence[SampleT],
+    ) -> list[SampleT]:
+        """Combine new samples with replay samples."""
+        new_ids = {self._get_sample_id(s) for s in new_samples}
+        replay = self.sample_replay(len(new_samples), exclude=new_ids)
+        return list(new_samples) + replay
+
+    def sample_replay(
+        self,
+        new_batch_size: int,
+        *,
+        exclude: Optional[Iterable[str]] = None,
+    ) -> list[SampleT]:
+        """
+        Sample from replay buffer.
+
+        Args:
+            new_batch_size: Size of new batch (replay size = batch_size * ratio)
+            exclude: Sample IDs to exclude from replay
+
+        Returns:
+            List of replay samples
+        """
+        if not self._buffer or self.replay_ratio <= 0:
+            return []
+
+        exclude = set(exclude or [])
+        available = [
+            sample for sample in self._buffer if self._get_sample_id(sample) not in exclude
+        ]
+        if not available:
+            return []
+
+        replay_size = max(1, int(new_batch_size * self.replay_ratio))
+        replay_size = min(replay_size, len(available))
+        return self._rng.sample(available, replay_size)
+
+    def buffer_snapshot(self) -> list[SampleT]:
+        """Return a copy of the current buffer."""
+        return list(self._buffer)
+
+    def buffer_summary(self) -> SelectionSummary:
+        """Get summary statistics for the buffer."""
+        return SelectionSummary(
+            total_samples=len(self._buffer),
+            selected_samples=len(self._buffer),
+            strategy=f"buffer:{self.selector.strategy}",
+        )
+
+    def clear(self) -> None:
+        """Clear the buffer and metrics."""
+        self._buffer = []
+        self._metrics = {}
+
+    def update_metrics(self, metrics: dict[str, float]) -> None:
+        """
+        Update importance metrics for samples in buffer.
+
+        Args:
+            metrics: Dict mapping sample_id to importance score
+        """
+        self._metrics.update(metrics)
+
+    def _get_sample_id(self, sample: SampleT) -> str:
+        """Get sample_id from sample (supports dict or object)."""
+        if isinstance(sample, dict):
+            return sample.get("sample_id", sample.get("dialog_id", str(id(sample))))
+        return getattr(sample, "sample_id", getattr(sample, "dialog_id", str(id(sample))))

sage/middleware/components/sage_sias/coreset_selector.py

@@ -0,0 +1,302 @@
+"""
+Coreset Selection for Efficient Training
+
+Implements lightweight coreset selection strategies that identify the most
+valuable samples for training, reducing computational cost while maintaining
+model quality.
+
+Strategies:
+    - loss_topk: Select samples with highest loss (most informative)
+    - diversity: Select samples maximizing coverage of feature space
+    - hybrid: Combination of loss-based and diversity-based selection
+    - random: Uniform random sampling (baseline)
+
+This is a core component of SIAS (Streaming Importance-Aware Agent System).
+"""
+
+from __future__ import annotations
+
+import math
+import random
+import re
+from collections import Counter
+from dataclasses import dataclass
+from typing import Any, Optional, Protocol, Sequence, runtime_checkable
+
+
+@dataclass(slots=True)
+class SelectionSummary:
+    """Summary statistics for a selection operation."""
+
+    total_samples: int
+    selected_samples: int
+    strategy: str
+
+
+@runtime_checkable
+class SampleProtocol(Protocol):
+    """Protocol for samples that can be used with CoresetSelector."""
+
+    @property
+    def sample_id(self) -> str:
+        """Unique identifier for the sample."""
+        ...
+
+    @property
+    def text(self) -> str:
+        """Text content of the sample."""
+        ...
+
+    @property
+    def metadata(self) -> dict[str, Any]:
+        """Metadata dictionary."""
+        ...
+
+
+# Type alias for any sample that implements the protocol
+SampleT = Any  # Should implement SampleProtocol
+
+
+class CoresetSelector:
+    """
+    Implements lightweight coreset selection strategies.
+
+    This class provides several strategies for selecting a representative
+    subset of samples from a larger dataset, optimizing for training efficiency.
+
+    Attributes:
+        strategy: Selection strategy ("loss_topk", "diversity", "hybrid", "random")
+        metric_key: Key in metadata to use for loss-based selection
+        diversity_temperature: Temperature for diversity scoring
+        random_seed: Seed for reproducibility
+
+    Example:
+        >>> selector = CoresetSelector(strategy="hybrid")
+        >>> selected = selector.select(samples, target_size=1000)
+        >>> print(f"Selected {len(selected)} from {len(samples)} samples")
+    """
+
+    STRATEGIES = ("loss_topk", "diversity", "hybrid", "random")
+
+    def __init__(
+        self,
+        strategy: str = "loss_topk",
+        metric_key: str = "loss",
+        diversity_temperature: float = 0.7,
+        random_seed: int = 13,
+    ) -> None:
+        """
+        Initialize CoresetSelector.
+
+        Args:
+            strategy: Selection strategy to use
+            metric_key: Metadata key for loss-based selection
+            diversity_temperature: Temperature for diversity scoring
+            random_seed: Random seed for reproducibility
+        """
+        if strategy not in self.STRATEGIES:
+            raise ValueError(f"Unknown strategy: {strategy}. Choose from {self.STRATEGIES}")
+
+        self.strategy = strategy
+        self.metric_key = metric_key
+        self.diversity_temperature = diversity_temperature
+        self._rng = random.Random(random_seed)
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+    def select(
+        self,
+        samples: Sequence[SampleT],
+        *,
+        target_size: Optional[int],
+        metrics: Optional[dict[str, float]] = None,
+    ) -> list[SampleT]:
+        """
+        Select a subset of samples using the configured strategy.
+
+        Args:
+            samples: Input samples to select from
+            target_size: Number of samples to select (None = keep all)
+            metrics: Optional external metrics dict mapping sample_id to score
+
+        Returns:
+            List of selected samples
+        """
+        if target_size is None or target_size <= 0 or target_size >= len(samples):
+            return list(samples)
+
+        if self.strategy == "loss_topk":
+            return self._select_loss(samples, target_size, metrics)
+        if self.strategy == "diversity":
+            return self._select_diversity(samples, target_size)
+        if self.strategy == "hybrid":
+            return self._select_hybrid(samples, target_size, metrics)
+        return self._select_random(samples, target_size)
+
+    def summary(self, original_size: int, selected_size: int) -> SelectionSummary:
+        """Create a summary of the selection operation."""
+        return SelectionSummary(
+            total_samples=original_size,
+            selected_samples=selected_size,
+            strategy=self.strategy,
+        )
+
+    # ------------------------------------------------------------------
+    # Selection Strategies
+    # ------------------------------------------------------------------
+    def _select_loss(
+        self,
+        samples: Sequence[SampleT],
+        target_size: int,
+        metrics: Optional[dict[str, float]],
+    ) -> list[SampleT]:
+        """Select samples with highest loss/importance scores."""
+
+        def score(sample: SampleT) -> float:
+            sample_id = self._get_sample_id(sample)
+            if metrics and sample_id in metrics:
+                return metrics[sample_id]
+            meta = self._get_metadata(sample)
+            meta_val = meta.get(self.metric_key)
+            if isinstance(meta_val, (int, float)):
+                return float(meta_val)
+            return 0.0
+
+        ranked = sorted(samples, key=score, reverse=True)
+        return list(ranked[:target_size])
+
+    def _select_random(
+        self,
+        samples: Sequence[SampleT],
+        target_size: int,
+    ) -> list[SampleT]:
+        """Uniform random sampling."""
+        return self._rng.sample(list(samples), target_size)
+
+    def _select_hybrid(
+        self,
+        samples: Sequence[SampleT],
+        target_size: int,
+        metrics: Optional[dict[str, float]],
+    ) -> list[SampleT]:
+        """Hybrid selection: 60% loss-based + 40% diversity-based."""
+        loss_portion = int(target_size * 0.6)
+        div_portion = target_size - loss_portion
+
+        # First select high-loss samples
+        top_loss = self._select_loss(samples, loss_portion or 1, metrics)
+        top_loss_ids = {self._get_sample_id(s) for s in top_loss}
+
+        # Then select diverse samples from remaining
+        remaining = [s for s in samples if self._get_sample_id(s) not in top_loss_ids]
+        if not remaining:
+            return top_loss
+
+        diversity = self._select_diversity(remaining, max(div_portion, 1))
+        merged = (top_loss + diversity)[:target_size]
+        return merged
+
+    def _select_diversity(
+        self,
+        samples: Sequence[SampleT],
+        target_size: int,
+    ) -> list[SampleT]:
+        """Select samples maximizing feature space coverage."""
+        if not samples:
+            return []
+
+        # Extract features for all samples
+        features = {
+            self._get_sample_id(sample): self._text_features(self._get_text(sample))
+            for sample in samples
+        }
+
+        selected: list[SampleT] = []
+        candidates = list(samples)
+
+        # Start with the sample that has the highest token variance
+        scores = {
+            self._get_sample_id(sample): self._feature_norm(features[self._get_sample_id(sample)])
+            for sample in samples
+        }
+        first = max(candidates, key=lambda s: scores.get(self._get_sample_id(s), 0.0))
+        selected.append(first)
+        candidates = [s for s in candidates if self._get_sample_id(s) != self._get_sample_id(first)]
+
+        # Iteratively select most diverse samples
+        while candidates and len(selected) < target_size:
+            best_candidate = max(
+                candidates,
+                key=lambda sample: self._min_distance(sample, selected, features),
+            )
+            selected.append(best_candidate)
+            candidates = [
+                s
+                for s in candidates
+                if self._get_sample_id(s) != self._get_sample_id(best_candidate)
+            ]
+
+        return selected
+
+    # ------------------------------------------------------------------
+    # Feature Extraction Helpers
+    # ------------------------------------------------------------------
+    def _text_features(self, text: str) -> Counter:
+        """Extract normalized token frequency features from text."""
+        tokens = re.findall(r"[a-zA-Z0-9_]+", text.lower())
+        filtered = [token for token in tokens if len(token) > 2]
+        counts = Counter(filtered)
+        total = sum(counts.values()) or 1.0
+        for key in counts:
+            counts[key] /= total
+        return counts
+
+    def _feature_norm(self, features: Counter) -> float:
+        """Compute L2 norm of feature vector."""
+        return math.sqrt(sum(value * value for value in features.values()))
+
+    def _cosine_similarity(self, left: Counter, right: Counter) -> float:
+        """Compute cosine similarity between two feature vectors."""
+        keys = left.keys() & right.keys()
+        if not keys:
+            return 0.0
+        return sum(left[key] * right[key] for key in keys)
+
+    def _min_distance(
+        self,
+        candidate: SampleT,
+        selected: Sequence[SampleT],
+        features: dict[str, Counter],
+    ) -> float:
+        """Compute minimum distance from candidate to selected set."""
+        cand_feat = features[self._get_sample_id(candidate)]
+        if not selected:
+            return 1.0
+        sims = [
+            self._cosine_similarity(cand_feat, features[self._get_sample_id(item)])
+            for item in selected
+        ]
+        similarity = max(sims) if sims else 0.0
+        return 1.0 - similarity
+
+    # ------------------------------------------------------------------
+    # Sample Access Helpers (support both dict and object access)
+    # ------------------------------------------------------------------
+    def _get_sample_id(self, sample: SampleT) -> str:
+        """Get sample_id from sample (supports dict or object)."""
+        if isinstance(sample, dict):
+            return sample.get("sample_id", sample.get("dialog_id", str(id(sample))))
+        return getattr(sample, "sample_id", getattr(sample, "dialog_id", str(id(sample))))
+
+    def _get_text(self, sample: SampleT) -> str:
+        """Get text from sample (supports dict or object)."""
+        if isinstance(sample, dict):
+            return sample.get("text", "")
+        return getattr(sample, "text", "")
+
+    def _get_metadata(self, sample: SampleT) -> dict[str, Any]:
+        """Get metadata from sample (supports dict or object)."""
+        if isinstance(sample, dict):
+            return sample.get("metadata", {})
+        return getattr(sample, "metadata", {})

sage/middleware/components/sage_sias/types.py

@@ -0,0 +1,94 @@
+"""
+SIAS Core Data Types
+
+Defines the core data structures used across SIAS components.
+These are designed to be independent of specific data sources.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Protocol, runtime_checkable
+
+
+@dataclass(slots=True)
+class SIASSample:
+    """
+    Generic sample container for SIAS algorithms.
+
+    This is a lightweight data class that can wrap samples from various sources.
+    The only required fields are sample_id and text; everything else is optional.
+
+    Attributes:
+        sample_id: Unique identifier for this sample
+        text: The text content (or serialized representation)
+        metadata: Arbitrary metadata dictionary
+        importance_score: SSIS-computed importance score (set during training)
+    """
+
+    sample_id: str
+    text: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+    importance_score: float = 0.0
+
+    def __hash__(self) -> int:
+        return hash(self.sample_id)
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, SIASSample):
+            return self.sample_id == other.sample_id
+        return False
+
+
+@runtime_checkable
+class SampleProtocol(Protocol):
+    """
+    Protocol for samples that can be used with SIAS algorithms.
+
+    Any class with these attributes can be used with CoresetSelector
+    and OnlineContinualLearner without modification.
+    """
+
+    @property
+    def sample_id(self) -> str:
+        """Unique identifier for the sample."""
+        ...
+
+    @property
+    def text(self) -> str:
+        """Text content of the sample."""
+        ...
+
+    @property
+    def metadata(self) -> dict[str, Any]:
+        """Metadata dictionary."""
+        ...
+
+
+# Backward compatibility alias
+# This allows existing code using ProcessedDialog to work with SIAS
+# by implementing the SampleProtocol
+Sample = SIASSample
+
+
+def wrap_sample(
+    sample_id: str,
+    text: str,
+    metadata: dict[str, Any] | None = None,
+    **kwargs: Any,
+) -> SIASSample:
+    """
+    Factory function to create a SIASSample.
+
+    Args:
+        sample_id: Unique identifier
+        text: Text content
+        metadata: Optional metadata dict
+        **kwargs: Additional metadata fields
+
+    Returns:
+        A new SIASSample instance
+    """
+    meta = metadata or {}
+    meta.update(kwargs)
+    return SIASSample(sample_id=sample_id, text=text, metadata=meta)

sage/middleware/components/sage_tsdb/__init__.py

@@ -0,0 +1,81 @@
+"""
+SAGE-TSDB: Time Series Database Component for SAGE
+
+Provides efficient time series data storage, querying, and processing capabilities
+for streaming and historical data analysis.
+
+Note: SAGE TSDB core is now an independent PyPI package (isage-tsdb).
+This module provides backward-compatible wrappers and SAGE-specific services.
+"""
+
+import warnings
+
+# Import from PyPI package (isage-tsdb)
+_SAGE_TSDB_AVAILABLE = False
+try:
+    from sage_tsdb import (
+        QueryConfig,
+        TimeRange,
+        TimeSeriesData,
+        TimeSeriesDB,
+        TimeSeriesIndex,
+    )
+
+    # Backward compatibility alias
+    SageTSDB = TimeSeriesDB
+    _SAGE_TSDB_AVAILABLE = True
+except ImportError as e:
+    # Don't fail immediately - allow graceful degradation
+    warnings.warn(
+        f"SAGE TSDB not available: {e}\n"
+        "Install with: pip install isage-tsdb\n"
+        "Time series features will be unavailable.",
+        UserWarning,
+        stacklevel=2,
+    )
+    # Provide stub exports
+    SageTSDB = None
+    TimeSeriesDB = None
+    TimeSeriesData = None
+    QueryConfig = None
+    TimeRange = None
+    TimeSeriesIndex = None
+
+# Algorithms (SAGE-specific extensions)
+# Only import if base package is available
+if _SAGE_TSDB_AVAILABLE:
+    from .python.algorithms import (
+        OutOfOrderStreamJoin,
+        TimeSeriesAlgorithm,
+        WindowAggregator,
+    )
+
+    # Micro-service wrapper (SAGE-specific)
+    from .python.micro_service.sage_tsdb_service import (
+        SageTSDBService,
+        SageTSDBServiceConfig,
+    )
+else:
+    # Stub classes if TSDB not available
+    TimeSeriesAlgorithm = None
+    OutOfOrderStreamJoin = None
+    WindowAggregator = None
+    SageTSDBService = None
+    SageTSDBServiceConfig = None
+
+__all__ = [
+    # Core API (may be None if not installed)
+    "SageTSDB",
+    "TimeSeriesData",
+    "QueryConfig",
+    "TimeRange",
+    # Service
+    "SageTSDBService",
+    "SageTSDBServiceConfig",
+    # Algorithms
+    "TimeSeriesAlgorithm",
+    "OutOfOrderStreamJoin",
+    "WindowAggregator",
+    # Availability flag
+    "_SAGE_TSDB_AVAILABLE",
+]

sage/middleware/components/sage_tsdb/python/__init__.py

@@ -0,0 +1,21 @@
+"""
+Python package for SageTSDB
+
+This package provides both high-performance C++ bindings and pure Python implementations
+for time series database operations.
+"""
+
+try:
+    # Try to import C++ bindings first
+    from . import _sage_tsdb
+
+    TSDB_BACKEND = "cpp"
+except ImportError:
+    # Fallback to pure Python implementation
+    _sage_tsdb = None
+    TSDB_BACKEND = "python"
+
+# Import Python APIs (these wrap C++ or pure Python implementations)
+from . import algorithms, sage_tsdb
+
+__all__ = ["sage_tsdb", "algorithms", "_sage_tsdb", "TSDB_BACKEND"]

sage/middleware/components/sage_tsdb/python/_sage_tsdb.pyi

@@ -0,0 +1,17 @@
+"""
+Type stub for SAGE TSDB C++ extension module.
+
+This is a compiled C++ extension module created via pybind11.
+The actual implementation is in C++, this file provides type hints for Python.
+"""
+
+# Basic type hints for the C++ extension
+# Add specific function/class signatures as needed when you know the API
+
+class SageTSDB:
+    """SAGE TSDB C++ extension interface"""
+
+    def __init__(self) -> None: ...
+    # Add more methods as needed
+
+# Add other exported symbols from the C++ module as needed