pyintake 0.0.1__py3-none-any.whl

pyintake/__init__.py

@@ -0,0 +1,83 @@
+"""PyIntake — Transport-agnostic data ingestion system.
+
+Collects, validates, deduplicates, and buffers events from any transport,
+then drains them into pluggable downstream consumers with priority ordering
+and backpressure signaling.
+"""
+
+from __future__ import annotations
+
+from pyintake._protocols import Consumer, MetricsCollector, TransportAdapter, Validator
+from pyintake._types import (
+    BackpressureSignal,
+    BufferEntry,
+    ConsumeResult,
+    IngestResult,
+    Priority,
+    ValidationOutcome,
+)
+from pyintake.buffer._dedup import SlidingWindowDedup
+from pyintake.buffer._priority_buffer import PriorityBuffer
+from pyintake.core._drain import AdaptiveDrainLoop
+from pyintake.core._engine import IngestionEngine
+from pyintake.core._rate_limiter import TokenBucketRateLimiter
+from pyintake.errors import (
+    AdapterError,
+    BufferFullError,
+    ConfigError,
+    ConsumerError,
+    DrainError,
+    DuplicateEventError,
+    ErrorCode,
+    PyIntakeError,
+    RateLimitedError,
+    ShutdownTimeoutError,
+    ValidationError,
+)
+from pyintake.event import IngestEvent
+from pyintake.metrics._collector import InMemoryMetricsCollector
+from pyintake.metrics._models import MetricsSnapshot, SourceMetrics
+from pyintake.validation._chain import ValidationChain
+
+__all__ = [
+    # Core event type
+    "IngestEvent",
+    # Value types and enums
+    "Priority",
+    "BackpressureSignal",
+    "IngestResult",
+    "ValidationOutcome",
+    "ConsumeResult",
+    "BufferEntry",
+    # Protocols
+    "TransportAdapter",
+    "Validator",
+    "Consumer",
+    "MetricsCollector",
+    # Engine and drain
+    "IngestionEngine",
+    "AdaptiveDrainLoop",
+    # Buffer
+    "PriorityBuffer",
+    "SlidingWindowDedup",
+    # Validation
+    "ValidationChain",
+    # Rate limiting
+    "TokenBucketRateLimiter",
+    # Metrics
+    "InMemoryMetricsCollector",
+    "MetricsSnapshot",
+    "SourceMetrics",
+    # Errors
+    "PyIntakeError",
+    "ErrorCode",
+    "BufferFullError",
+    "ValidationError",
+    "DuplicateEventError",
+    "RateLimitedError",
+    "AdapterError",
+    "DrainError",
+    "ConfigError",
+    "ConsumerError",
+    "ShutdownTimeoutError",
+]

pyintake/_protocols.py

@@ -0,0 +1,145 @@
+"""Protocol definitions for PyIntake pluggable interfaces.
+
+Protocols define the contracts for transport adapters, validators,
+consumers, and metrics collectors. Implementations can be swapped
+in via dependency injection without coupling to concrete classes.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
+
+if TYPE_CHECKING:
+    from pyintake._types import ConsumeResult, ValidationOutcome
+    from pyintake.event import IngestEvent
+    from pyintake.metrics._models import MetricsSnapshot
+
+
+# ---------------------------------------------------------------------------
+# Transport adapter protocol
+# ---------------------------------------------------------------------------
+
+
+@runtime_checkable
+class TransportAdapter(Protocol):
+    """Receives events from a transport and feeds them into IngestionEngine.
+
+    Implementations handle transport-specific concerns (Kafka consumer,
+    HTTP server, file watcher, etc.) and call engine.ingest() for each event.
+    """
+
+    async def start(self, engine: Any) -> None:
+        """Start the adapter and begin feeding events to the engine.
+
+        Args:
+            engine: The IngestionEngine to feed events into.
+        """
+        ...
+
+    async def stop(self) -> None:
+        """Stop the adapter and release transport resources."""
+        ...
+
+
+# ---------------------------------------------------------------------------
+# Validator protocol
+# ---------------------------------------------------------------------------
+
+
+@runtime_checkable
+class Validator(Protocol):
+    """Validates an IngestEvent before it enters the buffer.
+
+    Validators run in the ingestion path (before buffering), so they
+    must be fast and synchronous. Heavy validation should be done
+    downstream by consumers.
+    """
+
+    @property
+    def name(self) -> str:
+        """Human-readable name for metrics and logging."""
+        ...
+
+    def validate(self, event: IngestEvent) -> ValidationOutcome:
+        """Validate a single event.
+
+        Args:
+            event: The event to validate.
+
+        Returns:
+            Outcome indicating whether the event is valid.
+        """
+        ...
+
+
+# ---------------------------------------------------------------------------
+# Consumer protocol
+# ---------------------------------------------------------------------------
+
+
+@runtime_checkable
+class Consumer(Protocol):
+    """Downstream consumer that receives drained event batches.
+
+    Consumers are the output side of the ingestion pipeline. They
+    receive batches of validated, deduplicated events from the
+    drain loop. Examples: pystator orchestrator, pycharter validator,
+    file collector for offline analysis.
+    """
+
+    async def consume(self, batch: list[IngestEvent]) -> ConsumeResult:
+        """Process a batch of events.
+
+        Args:
+            batch: List of events drained from the buffer.
+
+        Returns:
+            Result indicating success/failure and events processed.
+        """
+        ...
+
+    async def close(self) -> None:
+        """Release any resources held by the consumer."""
+        ...
+
+
+# ---------------------------------------------------------------------------
+# Metrics collector protocol
+# ---------------------------------------------------------------------------
+
+
+@runtime_checkable
+class MetricsCollector(Protocol):
+    """Collects ingestion metrics programmatically.
+
+    All methods are synchronous and must be fast — they are called
+    in the hot ingestion path.
+    """
+
+    def record_accepted(self, event: IngestEvent) -> None:
+        """Record a successfully accepted event."""
+        ...
+
+    def record_rejected(self, event: IngestEvent, reason: str) -> None:
+        """Record a rejected event with the rejection reason."""
+        ...
+
+    def record_drained(self, count: int, sources: list[str] | None = None) -> None:
+        """Record that count events were drained to consumers."""
+        ...
+
+    def record_deduplicated(self, event: IngestEvent) -> None:
+        """Record a deduplicated (duplicate-rejected) event."""
+        ...
+
+    def record_shed(self, event: IngestEvent) -> None:
+        """Record an event shed due to buffer pressure."""
+        ...
+
+    def record_dead_letter(self, event: IngestEvent) -> None:
+        """Record an event that exceeded max retries."""
+        ...
+
+    def snapshot(self) -> MetricsSnapshot:
+        """Return a frozen point-in-time snapshot of all metrics."""
+        ...

pyintake/_types.py

@@ -0,0 +1,135 @@
+"""Core value types for PyIntake.
+
+Frozen dataclasses and enums used throughout the ingestion system.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from enum import IntEnum, StrEnum
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from pyintake.event import IngestEvent
+
+
+class Priority(IntEnum):
+    """Buffer priority lanes. Lower value = higher priority.
+
+    HIGH events are never shed under pressure (only rejected when
+    the buffer is completely full). LOW events are shed first.
+    """
+
+    HIGH = 0
+    NORMAL = 1
+    LOW = 2
+
+
+class BackpressureSignal(StrEnum):
+    """Signal to callers about current buffer health.
+
+    Transport adapters use this to decide how to react:
+    - HTTP: return 429
+    - Kafka: pause partitions
+    - WebSocket: send flow-control frame
+    """
+
+    NONE = "none"
+    SLOW_DOWN = "slow_down"
+    REJECT = "reject"
+
+
+@dataclass(frozen=True, slots=True)
+class IngestResult:
+    """Result returned from every ingest() call.
+
+    Attributes:
+        accepted: Whether the event was accepted into the buffer.
+        event_id: The event ID that was processed.
+        buffer_utilization: Current buffer utilization (0.0 - 1.0).
+        backpressure: Backpressure signal for the caller.
+        rejection_reason: Human-readable reason if rejected.
+    """
+
+    accepted: bool
+    event_id: str
+    buffer_utilization: float
+    backpressure: BackpressureSignal
+    rejection_reason: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to dictionary."""
+        return {
+            "accepted": self.accepted,
+            "event_id": self.event_id,
+            "buffer_utilization": self.buffer_utilization,
+            "backpressure": self.backpressure.value,
+            "rejection_reason": self.rejection_reason,
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class ValidationOutcome:
+    """Result of running a validator on an event.
+
+    Attributes:
+        is_valid: Whether the event passed validation.
+        reasons: Rejection reasons (empty if valid).
+        validator_name: Name of the validator that produced this outcome.
+    """
+
+    is_valid: bool
+    reasons: tuple[str, ...] = ()
+    validator_name: str = ""
+
+    @classmethod
+    def accept(cls) -> ValidationOutcome:
+        """Create an accepted outcome."""
+        return cls(is_valid=True)
+
+    @classmethod
+    def reject(
+        cls,
+        reasons: tuple[str, ...] | list[str],
+        validator_name: str = "",
+    ) -> ValidationOutcome:
+        """Create a rejected outcome."""
+        return cls(
+            is_valid=False,
+            reasons=tuple(reasons),
+            validator_name=validator_name,
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class ConsumeResult:
+    """Result from a downstream consumer processing a batch.
+
+    Attributes:
+        success: Whether the consumer processed the batch without errors.
+        processed: Number of events successfully processed.
+        errors: Error messages for failed events.
+    """
+
+    success: bool
+    processed: int
+    errors: tuple[str, ...] = ()
+
+
+@dataclass(slots=True)
+class BufferEntry:
+    """Mutable wrapper around IngestEvent for retry tracking in the buffer.
+
+    The IngestEvent is frozen/immutable, but we need to track retry state
+    as events are requeued on consumer failure.
+
+    Attributes:
+        event: The immutable IngestEvent.
+        retry_count: Number of times this event has been requeued.
+        first_buffered_at: When the event first entered the buffer.
+    """
+
+    event: IngestEvent
+    retry_count: int = 0
+    first_buffered_at: datetime = field(default_factory=lambda: datetime.now(UTC))

pyintake/adapters/__init__.py

@@ -0,0 +1,6 @@
+"""Transport adapters for PyIntake.
+
+Pluggable adapters for HTTP, Kafka, WebSocket, and file-based ingestion.
+"""
+
+from __future__ import annotations

pyintake/api/__init__.py

@@ -0,0 +1,3 @@
+"""REST API for PyIntake health, metrics, and HTTP ingestion."""
+
+from __future__ import annotations

pyintake/api/routes/__init__.py

@@ -0,0 +1,3 @@
+"""API route modules."""
+
+from __future__ import annotations

pyintake/api/routes/v1/__init__.py

@@ -0,0 +1,3 @@
+"""V1 API routes."""
+
+from __future__ import annotations

pyintake/buffer/__init__.py

@@ -0,0 +1,6 @@
+"""Buffer components for PyIntake.
+
+Priority-based in-memory buffering with shed-under-pressure and deduplication.
+"""
+
+from __future__ import annotations

pyintake/buffer/_dedup.py

@@ -0,0 +1,110 @@
+"""Sliding-window deduplication for PyIntake.
+
+Time-bounded AND count-bounded dedup using OrderedDict for O(1)
+insert/lookup with LRU eviction.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from collections import OrderedDict
+
+
+class SlidingWindowDedup:
+    """Sliding-window deduplication by event ID.
+
+    Maintains a bounded set of recently seen event IDs. An event is
+    considered a duplicate if its ID was seen within the window.
+
+    Attributes:
+        window_seconds: Time window for dedup (seconds).
+        max_ids: Maximum number of IDs to track (prevents unbounded memory).
+    """
+
+    def __init__(
+        self,
+        window_seconds: int = 300,
+        max_ids: int = 100_000,
+    ) -> None:
+        if window_seconds <= 0:
+            raise ValueError("window_seconds must be positive")
+        if max_ids <= 0:
+            raise ValueError("max_ids must be positive")
+
+        self._window_seconds = window_seconds
+        self._max_ids = max_ids
+        self._lock = threading.Lock()
+        self._seen: OrderedDict[str, float] = OrderedDict()
+
+    def is_duplicate(self, event_id: str) -> bool:
+        """Check if event_id was seen within the sliding window.
+
+        Thread-safe.
+
+        Args:
+            event_id: The event ID to check.
+
+        Returns:
+            True if the event is a duplicate.
+        """
+        with self._lock:
+            self._evict_expired()
+            return event_id in self._seen
+
+    def record(self, event_id: str) -> None:
+        """Record event_id with current timestamp.
+
+        Thread-safe. Evicts expired entries and overflow entries
+        beyond max_ids.
+
+        Args:
+            event_id: The event ID to record.
+        """
+        with self._lock:
+            now = time.monotonic()
+            self._evict_expired()
+
+            if event_id in self._seen:
+                self._seen.move_to_end(event_id)
+                self._seen[event_id] = now
+                return
+
+            self._seen[event_id] = now
+
+            while len(self._seen) > self._max_ids:
+                self._seen.popitem(last=False)
+
+    def _evict_expired(self) -> None:
+        """Remove entries older than window_seconds from the front.
+
+        Must be called while holding self._lock.
+        """
+        cutoff = time.monotonic() - self._window_seconds
+        while self._seen:
+            oldest_key = next(iter(self._seen))
+            if self._seen[oldest_key] < cutoff:
+                self._seen.popitem(last=False)
+            else:
+                break
+
+    @property
+    def size(self) -> int:
+        """Number of event IDs currently tracked."""
+        with self._lock:
+            return len(self._seen)
+
+    @property
+    def window_seconds(self) -> int:
+        """The time window for dedup."""
+        return self._window_seconds
+
+    @property
+    def max_ids(self) -> int:
+        """Maximum number of IDs tracked."""
+        return self._max_ids
+
+    def clear(self) -> None:
+        """Clear all tracked event IDs."""
+        with self._lock:
+            self._seen.clear()