isage-middleware 0.2.4.3__cp311-none-any.whl

sage/middleware/components/sage_tsdb/python/algorithms/__init__.py

@@ -0,0 +1,17 @@
+"""
+Algorithms for time series processing.
+
+This module provides a pluggable algorithm interface for various
+time series processing tasks including stream joins, aggregations,
+and complex event processing.
+"""
+
+from .base import TimeSeriesAlgorithm
+from .out_of_order_join import OutOfOrderStreamJoin
+from .window_aggregator import WindowAggregator
+
+__all__ = [
+    "TimeSeriesAlgorithm",
+    "OutOfOrderStreamJoin",
+    "WindowAggregator",
+]

sage/middleware/components/sage_tsdb/python/algorithms/base.py

@@ -0,0 +1,51 @@
+"""
+Base algorithm interface for time series processing.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from ..sage_tsdb import TimeSeriesData
+
+
+class TimeSeriesAlgorithm(ABC):
+    """
+    Base class for time series processing algorithms.
+
+    All algorithm implementations should inherit from this class and
+    implement the process method.
+    """
+
+    def __init__(self, config: dict[str, Any] | None = None):
+        """
+        Initialize algorithm.
+
+        Args:
+            config: Algorithm-specific configuration
+        """
+        self.config = config or {}
+
+    @abstractmethod
+    def process(self, data: list[TimeSeriesData], **kwargs) -> Any:
+        """
+        Process time series data.
+
+        Args:
+            data: Input time series data points
+            **kwargs: Additional algorithm-specific parameters
+
+        Returns:
+            Processed results (algorithm-specific format)
+        """
+        pass
+
+    def reset(self):  # noqa: B027
+        """Reset algorithm state (for stateful algorithms)"""
+        pass
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get algorithm statistics"""
+        return {}
+
+
+__all__ = ["TimeSeriesAlgorithm"]

sage/middleware/components/sage_tsdb/python/algorithms/out_of_order_join.py

@@ -0,0 +1,248 @@
+"""
+Out-of-Order Stream Join Algorithm
+
+This algorithm handles joining two time series streams that may arrive
+out of order, using windowing and buffering strategies.
+"""
+
+from collections import defaultdict
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+from ..sage_tsdb import TimeSeriesData
+from .base import TimeSeriesAlgorithm
+
+
+@dataclass
+class JoinConfig:
+    """Configuration for stream join"""
+
+    window_size: int  # milliseconds
+    max_delay: int  # maximum out-of-order delay (ms)
+    join_key: str | None = None  # tag key for join condition
+    join_predicate: Callable[[TimeSeriesData, TimeSeriesData], bool] | None = None
+
+
+class StreamBuffer:
+    """Buffer for managing out-of-order streams"""
+
+    def __init__(self, max_delay: int):
+        """
+        Initialize stream buffer.
+
+        Args:
+            max_delay: Maximum allowed delay (ms)
+        """
+        self.max_delay = max_delay
+        self.buffer: list[TimeSeriesData] = []
+        self.watermark = 0  # Current watermark timestamp
+
+    def add(self, data: TimeSeriesData):
+        """Add data to buffer"""
+        self.buffer.append(data)
+        self._update_watermark()
+
+    def add_batch(self, data_list: list[TimeSeriesData]):
+        """Add multiple data points to buffer"""
+        self.buffer.extend(data_list)
+        self._update_watermark()
+
+    def _update_watermark(self):
+        """Update watermark based on latest data"""
+        if self.buffer:
+            # Sort buffer by timestamp
+            self.buffer.sort(key=lambda x: x.timestamp)
+            # Watermark is the latest timestamp minus max delay
+            latest = self.buffer[-1].timestamp
+            self.watermark = latest - self.max_delay
+
+    def get_ready_data(self) -> list[TimeSeriesData]:
+        """Get data that's ready for processing (before watermark)"""
+        ready = [d for d in self.buffer if d.timestamp <= self.watermark]
+        # Remove ready data from buffer
+        self.buffer = [d for d in self.buffer if d.timestamp > self.watermark]
+        return ready
+
+    def size(self) -> int:
+        """Get buffer size"""
+        return len(self.buffer)
+
+
+class OutOfOrderStreamJoin(TimeSeriesAlgorithm):
+    """
+    Out-of-Order Stream Join Algorithm.
+
+    This algorithm joins two time series streams that may arrive out of order.
+    It uses windowing and watermarking to handle late data while maintaining
+    join correctness.
+
+    Features:
+    - Handles out-of-order data arrival
+    - Window-based join semantics
+    - Configurable watermarking for late data
+    - Support for custom join predicates
+    """
+
+    def __init__(self, config: dict[str, Any] | None = None):
+        """
+        Initialize stream join algorithm.
+
+        Args:
+            config: Configuration dictionary with:
+                - window_size: Join window size in milliseconds
+                - max_delay: Maximum out-of-order delay in milliseconds
+                - join_key: Optional tag key for equi-join
+                - join_predicate: Optional custom join predicate function
+        """
+        super().__init__(config)
+
+        self.window_size = self.config.get("window_size", 10000)  # 10 seconds
+        self.max_delay = self.config.get("max_delay", 5000)  # 5 seconds
+        self.join_key = self.config.get("join_key", None)
+        self.join_predicate = self.config.get("join_predicate", None)
+
+        # Buffers for two streams
+        self.left_buffer = StreamBuffer(self.max_delay)
+        self.right_buffer = StreamBuffer(self.max_delay)
+
+        # Statistics
+        self.stats = {
+            "total_joined": 0,
+            "late_arrivals": 0,
+            "dropped_late": 0,
+        }
+
+    def add_left_stream(self, data: list[TimeSeriesData]):
+        """Add data to left stream"""
+        self.left_buffer.add_batch(data)
+
+    def add_right_stream(self, data: list[TimeSeriesData]):
+        """Add data to right stream"""
+        self.right_buffer.add_batch(data)
+
+    def process(
+        self,
+        data: list[TimeSeriesData] | None = None,
+        left_stream: list[TimeSeriesData] | None = None,
+        right_stream: list[TimeSeriesData] | None = None,
+        **kwargs,
+    ) -> list[tuple[TimeSeriesData, TimeSeriesData]]:
+        """
+        Process stream join.
+
+        Args:
+            data: Not used (for compatibility)
+            left_stream: Data from left stream
+            right_stream: Data from right stream
+            **kwargs: Additional parameters
+
+        Returns:
+            List of joined data pairs
+        """
+        # Add data to buffers
+        if left_stream:
+            self.add_left_stream(left_stream)
+        if right_stream:
+            self.add_right_stream(right_stream)
+
+        # Get ready data from both buffers
+        left_ready = self.left_buffer.get_ready_data()
+        right_ready = self.right_buffer.get_ready_data()
+
+        # Perform join
+        joined = self._join_data(left_ready, right_ready)
+
+        # Update statistics
+        self.stats["total_joined"] += len(joined)
+
+        return joined
+
+    def _join_data(
+        self, left_data: list[TimeSeriesData], right_data: list[TimeSeriesData]
+    ) -> list[tuple[TimeSeriesData, TimeSeriesData]]:
+        """
+        Join data from two streams.
+
+        Args:
+            left_data: Data from left stream
+            right_data: Data from right stream
+
+        Returns:
+            List of joined pairs
+        """
+        joined = []
+
+        # If join key is specified, use hash join
+        if self.join_key:
+            joined = self._hash_join(left_data, right_data)
+        else:
+            # Use nested loop join with window condition
+            joined = self._nested_loop_join(left_data, right_data)
+
+        return joined
+
+    def _hash_join(
+        self, left_data: list[TimeSeriesData], right_data: list[TimeSeriesData]
+    ) -> list[tuple[TimeSeriesData, TimeSeriesData]]:
+        """Hash join on specified key"""
+        joined = []
+
+        # Build hash table for right stream
+        right_hash: dict[str, list[TimeSeriesData]] = defaultdict(list)
+        for right in right_data:
+            key_value = right.tags.get(self.join_key) if self.join_key else None
+            if key_value:
+                right_hash[key_value].append(right)
+
+        # Probe with left stream
+        for left in left_data:
+            key_value = left.tags.get(self.join_key) if self.join_key else None
+            if key_value and key_value in right_hash:
+                for right in right_hash[key_value]:
+                    # Check window condition
+                    if abs(left.timestamp - right.timestamp) <= self.window_size:
+                        # Check custom predicate if provided
+                        if self.join_predicate is None or self.join_predicate(left, right):
+                            joined.append((left, right))
+
+        return joined
+
+    def _nested_loop_join(
+        self, left_data: list[TimeSeriesData], right_data: list[TimeSeriesData]
+    ) -> list[tuple[TimeSeriesData, TimeSeriesData]]:
+        """Nested loop join with window condition"""
+        joined = []
+
+        for left in left_data:
+            for right in right_data:
+                # Check window condition
+                if abs(left.timestamp - right.timestamp) <= self.window_size:
+                    # Check custom predicate if provided
+                    if self.join_predicate is None or self.join_predicate(left, right):
+                        joined.append((left, right))
+
+        return joined
+
+    def reset(self):
+        """Reset algorithm state"""
+        self.left_buffer = StreamBuffer(self.max_delay)
+        self.right_buffer = StreamBuffer(self.max_delay)
+        self.stats = {
+            "total_joined": 0,
+            "late_arrivals": 0,
+            "dropped_late": 0,
+        }
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get join statistics"""
+        return {
+            **self.stats,
+            "left_buffer_size": self.left_buffer.size(),
+            "right_buffer_size": self.right_buffer.size(),
+            "left_watermark": self.left_buffer.watermark,
+            "right_watermark": self.right_buffer.watermark,
+        }
+
+
+__all__ = ["OutOfOrderStreamJoin", "JoinConfig", "StreamBuffer"]

sage/middleware/components/sage_tsdb/python/algorithms/window_aggregator.py

@@ -0,0 +1,296 @@
+"""
+Window Aggregator Algorithm
+
+Provides various windowing strategies for time series aggregation,
+including tumbling, sliding, and session windows.
+"""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any
+
+import numpy as np
+
+from ..sage_tsdb import AggregationType, TimeSeriesData
+from .base import TimeSeriesAlgorithm
+
+
+class WindowType(Enum):
+    """Window types for aggregation"""
+
+    TUMBLING = "tumbling"  # Non-overlapping fixed-size windows
+    SLIDING = "sliding"  # Overlapping fixed-size windows
+    SESSION = "session"  # Dynamic windows based on inactivity gap
+
+
+@dataclass
+class WindowConfig:
+    """Configuration for windowing"""
+
+    window_type: WindowType
+    window_size: int  # milliseconds
+    slide_interval: int | None = None  # for sliding windows (ms)
+    session_gap: int | None = None  # for session windows (ms)
+    aggregation: AggregationType = AggregationType.AVG
+
+
+class WindowAggregator(TimeSeriesAlgorithm):
+    """
+    Window-based aggregation algorithm.
+
+    Supports multiple windowing strategies:
+    - Tumbling windows: Non-overlapping fixed-size windows
+    - Sliding windows: Overlapping windows with configurable slide interval
+    - Session windows: Dynamic windows based on inactivity gaps
+
+    Features:
+    - Multiple aggregation functions (sum, avg, min, max, count, etc.)
+    - Efficient incremental computation
+    - Support for late data handling
+    """
+
+    def __init__(self, config: dict[str, Any] | None = None):
+        """
+        Initialize window aggregator.
+
+        Args:
+            config: Configuration dictionary with:
+                - window_type: Type of window (tumbling/sliding/session)
+                - window_size: Window size in milliseconds
+                - slide_interval: Slide interval for sliding windows (ms)
+                - session_gap: Inactivity gap for session windows (ms)
+                - aggregation: Aggregation function to apply
+        """
+        super().__init__(config)
+
+        window_type_str = self.config.get("window_type", "tumbling")
+        self.window_type = WindowType(window_type_str)
+        self.window_size = self.config.get("window_size", 60000)  # 1 minute
+        self.slide_interval = self.config.get("slide_interval", self.window_size)
+        self.session_gap = self.config.get("session_gap", 30000)  # 30 seconds
+
+        agg_str = self.config.get("aggregation", "avg")
+        if isinstance(agg_str, str):
+            self.aggregation = AggregationType(agg_str)
+        else:
+            self.aggregation = agg_str
+
+        # State for incremental processing
+        self.windows: dict[int, list[TimeSeriesData]] = {}
+        self.stats = {
+            "windows_created": 0,
+            "windows_completed": 0,
+            "data_points_processed": 0,
+        }
+
+    def process(self, data: list[TimeSeriesData], **kwargs) -> list[TimeSeriesData]:
+        """
+        Process time series data with windowing.
+
+        Args:
+            data: Input time series data points
+            **kwargs: Additional parameters
+
+        Returns:
+            Aggregated time series data (one point per window)
+        """
+        if not data:
+            return []
+
+        # Sort data by timestamp
+        sorted_data = sorted(data, key=lambda x: x.timestamp)
+
+        # Apply windowing based on type
+        if self.window_type == WindowType.TUMBLING:
+            return self._tumbling_window(sorted_data)
+        elif self.window_type == WindowType.SLIDING:
+            return self._sliding_window(sorted_data)
+        elif self.window_type == WindowType.SESSION:
+            return self._session_window(sorted_data)
+
+        return []
+
+    def _tumbling_window(self, data: list[TimeSeriesData]) -> list[TimeSeriesData]:
+        """Process with tumbling windows"""
+        if not data:
+            return []
+
+        results = []
+        window_start = self._align_to_window(data[0].timestamp)
+        window_data = []
+
+        for point in data:
+            window_key = self._get_window_key(point.timestamp, window_start)
+
+            # Check if point belongs to current window
+            if window_key == window_start:
+                window_data.append(point)
+            else:
+                # Complete current window
+                if window_data:
+                    agg_point = self._aggregate_window(window_data, window_start)
+                    results.append(agg_point)
+                    self.stats["windows_completed"] += 1
+
+                # Start new window(s)
+                # Handle potential gaps
+                while window_key > window_start:
+                    window_start += self.window_size
+
+                window_data = [point]
+                self.stats["windows_created"] += 1
+
+        # Complete last window
+        if window_data:
+            agg_point = self._aggregate_window(window_data, window_start)
+            results.append(agg_point)
+            self.stats["windows_completed"] += 1
+
+        self.stats["data_points_processed"] += len(data)
+        return results
+
+    def _sliding_window(self, data: list[TimeSeriesData]) -> list[TimeSeriesData]:
+        """Process with sliding windows"""
+        if not data:
+            return []
+
+        results = []
+
+        # Get first window start
+        first_timestamp = data[0].timestamp
+        window_start = self._align_to_window(first_timestamp)
+
+        # Create windows until we've covered all data
+        last_timestamp = data[-1].timestamp
+
+        while window_start <= last_timestamp:
+            window_end = window_start + self.window_size
+
+            # Get data points in this window
+            window_data = [point for point in data if window_start <= point.timestamp < window_end]
+
+            if window_data:
+                agg_point = self._aggregate_window(window_data, window_start)
+                results.append(agg_point)
+                self.stats["windows_completed"] += 1
+
+            # Slide to next window
+            window_start += self.slide_interval
+            self.stats["windows_created"] += 1
+
+        self.stats["data_points_processed"] += len(data)
+        return results
+
+    def _session_window(self, data: list[TimeSeriesData]) -> list[TimeSeriesData]:
+        """Process with session windows"""
+        if not data:
+            return []
+
+        results = []
+        session_data = []
+        last_timestamp = data[0].timestamp
+        session_start = data[0].timestamp
+
+        for point in data:
+            # Check if point is within session gap
+            if point.timestamp - last_timestamp <= self.session_gap:
+                session_data.append(point)
+            else:
+                # Complete current session
+                if session_data:
+                    agg_point = self._aggregate_window(session_data, session_start)
+                    results.append(agg_point)
+                    self.stats["windows_completed"] += 1
+
+                # Start new session
+                session_data = [point]
+                session_start = point.timestamp
+                self.stats["windows_created"] += 1
+
+            last_timestamp = point.timestamp
+
+        # Complete last session
+        if session_data:
+            agg_point = self._aggregate_window(session_data, session_start)
+            results.append(agg_point)
+            self.stats["windows_completed"] += 1
+
+        self.stats["data_points_processed"] += len(data)
+        return results
+
+    def _align_to_window(self, timestamp: int) -> int:
+        """Align timestamp to window boundary"""
+        return (timestamp // self.window_size) * self.window_size
+
+    def _get_window_key(self, timestamp: int, reference: int) -> int:
+        """Get window key for timestamp"""
+        return self._align_to_window(timestamp)
+
+    def _aggregate_window(
+        self, data: list[TimeSeriesData], window_timestamp: int
+    ) -> TimeSeriesData:
+        """Aggregate data in a window"""
+        if not data:
+            return TimeSeriesData(timestamp=window_timestamp, value=0.0)
+
+        # Extract values
+        values = []
+        for point in data:
+            # Flatten arrays/lists, append scalars
+            if isinstance(point.value, (list, np.ndarray)):
+                # Use np.ravel to flatten, then convert to list and extend
+                values.extend(np.ravel(point.value).tolist())
+            else:
+                values.append(point.value)
+
+        # Apply aggregation
+        if self.aggregation == AggregationType.SUM:
+            agg_value = sum(values)
+        elif self.aggregation == AggregationType.AVG:
+            agg_value = sum(values) / len(values)
+        elif self.aggregation == AggregationType.MIN:
+            agg_value = min(values)
+        elif self.aggregation == AggregationType.MAX:
+            agg_value = max(values)
+        elif self.aggregation == AggregationType.COUNT:
+            agg_value = len(values)
+        elif self.aggregation == AggregationType.FIRST:
+            agg_value = values[0]
+        elif self.aggregation == AggregationType.LAST:
+            agg_value = values[-1]
+        elif self.aggregation == AggregationType.STDDEV:
+            agg_value = float(np.std(values))
+        else:
+            agg_value = sum(values) / len(values)
+
+        # Merge tags from all data points
+        merged_tags = {}
+        for point in data:
+            if point.tags:
+                merged_tags.update(point.tags)
+
+        return TimeSeriesData(
+            timestamp=window_timestamp,
+            value=agg_value,
+            tags=merged_tags,
+            fields={"window_size": len(data), "aggregation": self.aggregation.value},
+        )
+
+    def reset(self):
+        """Reset algorithm state"""
+        self.windows = {}
+        self.stats = {
+            "windows_created": 0,
+            "windows_completed": 0,
+            "data_points_processed": 0,
+        }
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get aggregator statistics"""
+        return {
+            **self.stats,
+            "active_windows": len(self.windows),
+        }
+
+
+__all__ = ["WindowAggregator", "WindowType", "WindowConfig"]

sage/middleware/components/sage_tsdb/python/micro_service/__init__.py

@@ -0,0 +1,7 @@
+"""
+Micro-service module for SageTSDB
+"""
+
+from .sage_tsdb_service import SageTSDBService, SageTSDBServiceConfig
+
+__all__ = ["SageTSDBService", "SageTSDBServiceConfig"]