velm 0.1.2__py3-none-any.whl

velm/core/daemon/transporter/buffer.py

@@ -0,0 +1,207 @@
+# Path: scaffold/core/daemon/transporter/buffer.py
+# ------------------------------------------------
+# LIF: INFINITY | AUTH_CODE: Ω_BUFFER_PYTHON_TYPE_SAFE_V999
+# SYSTEM: KINETIC_STORAGE | ROLE: MEMORY_VESSEL
+# =================================================================================
+# == THE KINETIC BUFFER (V-Ω-HYDRODYNAMIC-TYPE-SAFE)                             ==
+# =================================================================================
+
+import logging
+from typing import Optional, Dict, Any
+from .constants import (
+    HEADER_TERMINATOR,
+    HEADER_REGEX,
+    MAX_PAYLOAD_SIZE,
+    BUFFER_SAFETY_CAP,
+    PARANOID_LOGGING
+)
+
+Logger = logging.getLogger("KineticBuffer")
+
+
+class KineticBuffer:
+    """
+    The High-Performance Memory Vessel (Python Edition).
+    Wraps a mutable `bytearray` with safety valves and forensic instrumentation.
+    Strictly handles bytes/bytearray types to prevent TypeError fractures.
+    """
+    __slots__ = ['_data', '_expected_length', '_total_ingested', '_peak_size', '_resync_count']
+
+    def __init__(self):
+        self._data = bytearray()
+        self._expected_length: Optional[int] = None
+        self._total_ingested = 0
+        self._peak_size = 0
+        self._resync_count = 0
+
+    def feed(self, chunk: bytes):
+        """
+        [RITE OF INGESTION]
+        Adds raw matter to the vessel. Safe against overflows.
+        """
+        try:
+            if not chunk: return
+
+            self._data.extend(chunk)
+            self._total_ingested += len(chunk)
+
+            current_len = len(self._data)
+            if current_len > self._peak_size:
+                self._peak_size = current_len
+
+            # [ASCENSION 4]: FLOOD GATE PROTECTION
+            if current_len > BUFFER_SAFETY_CAP:
+                # If we are not tracking a body, and we have no header, we are drowning.
+                # HEADER_TERMINATOR is bytes, _data is bytearray. Safe.
+                if self._expected_length is None and HEADER_TERMINATOR not in self._data:
+                    if PARANOID_LOGGING:
+                        Logger.critical(f"🌊 Buffer Flood ({current_len} bytes). Resynchronizing.")
+                    self._perform_resync()
+
+        except Exception as e:
+            Logger.error(f"FEED FRACTURE: {e}")
+            self.clear()
+
+    def try_extract_frame(self) -> Optional[bytes]:
+        """
+        [RITE]: EXTRACT_LSP_FRAME
+        Attempts to slice a valid LSP message from the buffer.
+        Returns bytes if successful, None if hungry.
+        """
+        try:
+            # PHASE 1: HEADER RESOLUTION
+            if self._expected_length is None:
+                # HEADER_TERMINATOR is b'\r\n\r\n'. Safe for bytearray.find().
+                sep_index = self._data.find(HEADER_TERMINATOR)
+
+                if sep_index == -1:
+                    return None  # Hunger
+
+                # Decode Header (ASCII)
+                try:
+                    # Slice is a bytearray, decode turns it to str for logging/regex
+                    header_bytes = self._data[:sep_index]
+
+                    # Regex search on BYTES (rb'...')
+                    match = HEADER_REGEX.search(header_bytes)
+                except Exception:
+                    if PARANOID_LOGGING: Logger.warning("Header Analysis Failed. Resyncing.")
+                    self._perform_resync(skip_to=sep_index + 4)
+                    return None
+
+                if not match:
+                    if PARANOID_LOGGING: Logger.warning("Header missing Content-Length. Resyncing.")
+                    self._perform_resync(skip_to=sep_index + 4)
+                    return self.try_extract_frame()  # Recursive check
+
+                try:
+                    # match.group(1) returns bytes, int() handles bytes correctly
+                    content_len = int(match.group(1))
+                except ValueError:
+                    self._perform_resync()
+                    return None
+
+                if content_len < 0 or content_len > MAX_PAYLOAD_SIZE:
+                    Logger.error(f"Profane Content-Length {content_len}. Purging.")
+                    self._perform_resync()
+                    return None
+
+                self._expected_length = content_len
+
+                # Slide Buffer past Header (O(n) memmove in CPython)
+                # +4 for len(b'\r\n\r\n')
+                del self._data[:sep_index + 4]
+
+            # PHASE 2: BODY EXTRACTION
+            if self._expected_length is not None:
+                if len(self._data) >= self._expected_length:
+                    # Extract Payload (Copy to bytes to make it immutable)
+                    payload = bytes(self._data[:self._expected_length])
+
+                    # Slide Buffer past Body
+                    del self._data[:self._expected_length]
+
+                    # Reset State
+                    self._expected_length = None
+
+                    return payload
+
+            return None  # Wait for more mass
+
+        except Exception as e:
+            Logger.error(f"FRAME EXTRACT FRACTURE: {e}")
+            self._perform_resync()
+            return None
+
+    def try_extract_raw_json(self) -> Optional[bytes]:
+        """
+        [RITE]: EXTRACT_RAW_DIALECT
+        Attempts to find a newline-delimited JSON object.
+        """
+        try:
+            # Search for byte newline
+            newline_idx = self._data.find(b'\n')
+
+            if newline_idx != -1:
+                # Extract the line
+                line = self._data[:newline_idx]
+
+                # Advance buffer past newline
+                del self._data[:newline_idx + 1]
+
+                # Check for JSON signature '{' (ASCII 123)
+                # Stripping whitespace from bytes
+                stripped = line.strip()
+                if stripped and stripped[0] == 123:
+                    return bytes(stripped)
+
+                # If it's garbage, we consumed it. Return None.
+                return None
+
+            return None  # Wait for newline
+        except Exception:
+            return None
+
+    def peek_byte(self) -> int:
+        """
+        Returns the integer value of the first byte (0-255) or -1 if empty.
+        Safe for comparison with ASCII integers.
+        """
+        if not self._data: return -1
+        return self._data[0]  # bytearray indexing returns int
+
+    def _perform_resync(self, skip_to: int = 0):
+        """Aggressively discards data until a new Header boundary is found."""
+        self._resync_count += 1
+
+        # If specific skip requested
+        if skip_to > 0:
+            del self._data[:skip_to]
+            self._expected_length = None
+            return
+
+        # Search for next terminator
+        next_header = self._data.find(HEADER_TERMINATOR)
+        if next_header != -1:
+            # Jump to start of next header
+            del self._data[:next_header]
+            self._expected_length = None
+        else:
+            # If no header found in entire buffer, nuke everything.
+            self.clear()
+
+    def clear(self):
+        del self._data[:]
+        self._expected_length = None
+
+    def has_data(self) -> bool:
+        return len(self._data) > 0
+
+    @property
+    def metrics(self) -> Dict[str, Any]:
+        return {
+            "ingested": self._total_ingested,
+            "peak": self._peak_size,
+            "resyncs": self._resync_count,
+            "current": len(self._data)
+        }

velm/core/daemon/transporter/constants.py

@@ -0,0 +1,37 @@
+# Path: scaffold/core/daemon/transporter/constants.py
+# ---------------------------------------------------
+# LIF: INFINITY | AUTH_CODE: Ω_PY_CONSTANTS_BINARY_V999
+# SYSTEM: KINETIC_PHYSICS | ROLE: IMMUTABLE_LAWS
+# =================================================================================
+# == THE LAWS OF KINETIC TRANSFER (PYTHON: BINARY EDITION)                       ==
+# =================================================================================
+
+import re
+
+# [PROTOCOL PHYSICS]
+# The dimensional barrier between Meta-Data (Header) and Gnosis (Body).
+# [CRITICAL]: Must be bytes (b'') to interact with socket data.
+HEADER_TERMINATOR = b'\r\n\r\n'
+
+# The Regex to divine the mass of the payload from the header.
+# Compiled once at module level for O(1) access.
+# [CRITICAL]: Pattern must be bytes (rb'') to search bytearrays.
+HEADER_REGEX = re.compile(rb'Content-Length: (\d+)', re.IGNORECASE)
+
+# [MEMORY ALLOCATION]
+# 64KB - Aligns with L1 CPU Cache pages for max throughput.
+DEFAULT_CHUNK_SIZE = 65536
+
+# 64MB - The maximum allowed thought size.
+MAX_PAYLOAD_SIZE = 64 * 1024 * 1024
+
+# Safety Valve: 1.2x Max Payload. If buffer exceeds this without header, PURGE.
+BUFFER_SAFETY_CAP = int(MAX_PAYLOAD_SIZE * 1.2)
+
+# [SOCKET PHYSICS]
+# Kernel buffer size target (1MB) to prevent syscall blocking.
+KERNEL_BUFFER_SIZE = 1048576
+
+# [FORENSICS]
+# Toggle this to True to flood stdout with hex dumps on error (Use with caution)
+PARANOID_LOGGING = True

velm/core/daemon/transporter/engine.py

@@ -0,0 +1,304 @@
+# Path: core/daemon/transporter/engine.py
+# ---------------------------------------
+import socket
+import json
+import logging
+import time
+import select
+import threading
+import errno
+import struct
+from collections import deque
+from typing import Generator, Dict, Any, Optional
+
+try:
+    from ..serializer import gnostic_serializer
+except ImportError:
+    gnostic_serializer = lambda x: str(x)
+
+Logger = logging.getLogger("GnosticTransporter")
+
+# [PHYSICS CONSTANTS - ASCENDED]
+HEADER_TERMINATOR = b'\r\n\r\n'
+
+# [ASCENSION 1]: HYPER-VELOCITY CHUNK (4MB)
+# Aligns with modern OS TCP window scaling for maximum single-syscall throughput.
+DEFAULT_CHUNK_SIZE = 4 * 1024 * 1024
+
+# [ASCENSION 2]: GOD-TIER PAYLOAD CAP (100MB)
+# Ensures even the largest "Plugin Census" or "Heresy Storm" can pass atomic validation.
+MAX_PAYLOAD_SIZE = 100 * 1024 * 1024
+
+# [ASCENSION 3]: DEEP OCEAN BUFFER (256MB)
+# Backpressure threshold raised significantly to absorb bursts without shedding logs.
+MAX_QUEUE_BYTES = 256 * 1024 * 1024
+
+# [ASCENSION 4]: INFINITE LOG HISTORY (5000 Lines)
+MAX_LOW_PRIORITY_DEPTH = 5000
+
+
+class GnosticTransporter:
+    """
+    =================================================================================
+    == THE INVINCIBLE TRANSPORTER (V-Ω-HYDRODYNAMIC-TITANIUM-V5-GOD-MODE)          ==
+    =================================================================================
+    LIF: INFINITY
+
+    The Physics Engine of the Gnostic Daemon. Handles the atomic movement of bytes
+    across the dimensional gap (Socket) using generator-based streaming and
+    zero-copy slicing.
+
+    ### THE PANTHEON OF 24 LEGENDARY ASCENSIONS:
+
+    1.  **Zero-Copy Egress:** Uses `memoryview` slicing to prevent RAM duplication.
+    2.  **Dual-Lane Priority:** Separates RPC (High) from Logs (Low).
+    3.  **Hydrodynamic Backpressure:** Sheds logs only when the 256MB buffer fills.
+    4.  **The Lazarus Catch:** Handles `WinError 10053/10058` gracefully.
+    5.  **The Sentinel Event:** Wake-on-write event eliminates polling latency.
+    6.  **Atomic Serialization:** JSON encoding occurs outside the write lock.
+    7.  **Hyper-Fast Select:** 10ms poll cycle for near-real-time responsiveness.
+    8.  **Dialect Auto-Detection:** Switches between JSON-RPC and Raw JSON instantly.
+    9.  **Forensic Telemetry:** Deep metric tracking for every byte moved.
+    10. **Thread Identity:** Named threads (`KineticPump`) for debugging.
+    11. **Partial Write Recovery:** Loops on `EWOULDBLOCK` to guarantee frame delivery.
+    12. **Idempotent Closure:** Safe shutdown logic prevents double-free errors.
+    13. **Hyper-Velocity Chunking:** 4MB read/write operations.
+    14. **Direct Binary Injection:** `send_raw` bypasses serialization for speed.
+    15. **Flood Gate Protection:** Hard limits on payload size to prevent DoS.
+    16. **GIL Release Strategy:** Micro-sleeps prevent CPU starvation.
+    17. **Buffer Pre-Allocation:** Minimizes memory fragmentation.
+    18. **Socket Physics Tuning:** Applies `TCP_NODELAY` and `SO_SNDBUF` optimizations.
+    19. **Fragment Reassembly:** Robust buffer handling for split packets.
+    20. **Error Sarcophagus:** Catches and logs IO errors without crashing the thread.
+    21. **Queue Draining:** Ensures high-priority packets flush before death.
+    22. **Keep-Alive Awareness:** Updates `_is_alive` state atomically.
+    23. **Type-Safe Signatures:** Full Python typing for reliability.
+    24. **Memory Leak Protection:** Explicit clearing of deques on shutdown.
+    """
+    __slots__ = [
+        'sock', '_buffer', '_high_queue', '_low_queue', '_write_lock',
+        '_is_alive', 'addr', 'metrics', 'id', '_writer_thread',
+        '_queue_byte_size', '_data_ready'
+    ]
+
+    def __init__(self, sock: socket.socket):
+        from .buffer import KineticBuffer
+        from .physics import SocketPhysics
+
+        self.sock = sock
+        self._buffer = KineticBuffer()
+        self._high_queue = deque()
+        self._low_queue = deque()
+        self._queue_byte_size = 0
+        self._write_lock = threading.RLock()
+        self._is_alive = True
+        self._data_ready = threading.Event()
+
+        try:
+            self.addr = sock.getpeername()
+            self.id = f"{self.addr[0]}:{self.addr[1]}"
+            self.sock.setblocking(False)
+        except OSError:
+            self.addr = ('unknown', 0)
+            self.id = "unknown:0"
+
+        self.metrics = {
+            "bytes_in": 0, "bytes_out": 0,
+            "msgs_in": 0, "msgs_out": 0,
+            "errors": 0, "backpressure_events": 0,
+            "dropped_packets": 0, "queue_pressure": 0
+        }
+
+        SocketPhysics.tune(self.sock)
+
+        self._writer_thread = threading.Thread(
+            target=self._hydrodynamic_pump_loop,
+            name=f"KineticPump-{self.id}",
+            daemon=True
+        )
+        self._writer_thread.start()
+
+    @property
+    def is_alive(self) -> bool:
+        return self._is_alive
+
+    def stream_messages(self) -> Generator[Dict[str, Any], None, None]:
+        """
+        [THE INGRESS RITE]
+        Reads from the socket, handles fragmentation, and yields complete JSON objects.
+        """
+        while self._is_alive:
+            try:
+                # 1. Wait for Data (Non-Blocking Select)
+                try:
+                    readable, _, exceptional = select.select([self.sock], [], [self.sock], 0.02)
+                except (select.error, ValueError, OSError):
+                    self._abort("Socket Invalidated")
+                    return
+
+                if exceptional:
+                    self._abort("Hardware exception")
+                    return
+
+                if readable:
+                    try:
+                        # [ASCENSION 13]: MASSIVE READ
+                        chunk = self.sock.recv(DEFAULT_CHUNK_SIZE)
+                        if not chunk:
+                            self._abort("Peer issued FIN")
+                            return
+                        self._buffer.feed(chunk)
+                        self.metrics["bytes_in"] += len(chunk)
+                    except BlockingIOError:
+                        pass
+                    except OSError as e:
+                        if getattr(e, 'winerror', 0) in (10053, 10054, 10058):
+                            self._abort(f"Link Severed: {e}")
+                            return
+                        raise
+
+                    # 2. Transmute Buffer to JSON
+                    while self._buffer.has_data():
+                        fb = self._buffer.peek_byte()
+                        # [ASCENSION 8]: DIALECT TRIAGE
+                        payload = self._buffer.try_extract_raw_json() if fb == 123 else self._buffer.try_extract_frame()
+
+                        if payload:
+                            self.metrics["msgs_in"] += 1
+                            try:
+                                decoded = json.loads(payload.decode('utf-8', errors='replace'))
+                                yield decoded
+                            except json.JSONDecodeError:
+                                Logger.error(f"[{self.id}] Malformed JSON. Dropping.")
+                        else:
+                            break
+            except Exception:
+                if self._is_alive:
+                    self.metrics["errors"] += 1
+                time.sleep(0.1)
+
+    def send_message(self, msg: Dict[str, Any], priority: bool = False) -> bool:
+        """
+        [THE EGRESS RITE]
+        Serializes and queues a message for the Pump. Non-blocking.
+        """
+        if not self._is_alive: return False
+        try:
+            # [ASCENSION 6]: ATOMIC SERIALIZATION
+            body = json.dumps(msg, default=gnostic_serializer).encode('utf-8')
+            header = f"Content-Length: {len(body)}\r\n\r\n".encode('ascii')
+            full_frame = header + body
+            return self.send_raw(full_frame, priority)
+        except Exception:
+            self.metrics["errors"] += 1
+            return False
+
+    def send_raw(self, frame: bytes, priority: bool = False) -> bool:
+        """
+        [ASCENSION 14]: DIRECT BINARY INJECTION
+        Allows pre-serialized frames to bypass local serialization logic.
+        """
+        if not self._is_alive: return False
+        try:
+            frame_len = len(frame)
+            with self._write_lock:
+                # [ASCENSION 3]: HYDRODYNAMIC BACKPRESSURE (256MB Cap)
+                if self._queue_byte_size + frame_len > MAX_QUEUE_BYTES and not priority:
+                    self.metrics["dropped_packets"] += 1
+                    self.metrics["backpressure_events"] += 1
+                    return True  # Silent drop to save the pipe
+
+                if priority:
+                    self._high_queue.append(frame)
+                else:
+                    if len(self._low_queue) > MAX_LOW_PRIORITY_DEPTH:
+                        self._low_queue.popleft()  # Shed oldest logs
+                    self._low_queue.append(frame)
+
+                self._queue_byte_size += frame_len
+
+            # [ASCENSION 5]: WAKE THE PUMP
+            self._data_ready.set()
+            return True
+        except Exception:
+            self.metrics["errors"] += 1
+            return False
+
+    def _hydrodynamic_pump_loop(self):
+        """
+        [THE KINETIC PUMP]
+        Runs in a background thread. Manages the physical write calls.
+        """
+        threading.current_thread().name = f"Pump-{self.id}"
+
+        while self._is_alive:
+            try:
+                packet = None
+                with self._write_lock:
+                    if self._high_queue:
+                        packet = self._high_queue.popleft()
+                    elif self._low_queue:
+                        packet = self._low_queue.popleft()
+
+                    if packet:
+                        self._queue_byte_size -= len(packet)
+
+                if not packet:
+                    # Wait for signal or timeout
+                    self._data_ready.wait(timeout=0.05)
+                    self._data_ready.clear()
+                    continue
+
+                total_sent = 0
+                frame_len = len(packet)
+                mv = memoryview(packet)
+
+                while total_sent < frame_len and self._is_alive:
+                    try:
+                        # [ASCENSION 7]: FAST WRITE SELECT
+                        _, writable, _ = select.select([], [self.sock], [], 0.01)
+                        if writable:
+                            # [ASCENSION 13]: MASSIVE WRITE
+                            sent = self.sock.send(mv[total_sent:])
+                            total_sent += sent
+                            self.metrics["bytes_out"] += sent
+                        else:
+                            time.sleep(0.001)  # Yield CPU
+                    except OSError as e:
+                        if e.errno == errno.EWOULDBLOCK or e.errno == errno.EAGAIN:
+                            time.sleep(0.001)
+                            continue
+                        # [ASCENSION 4]: WINDOWS ERROR HANDLING
+                        if getattr(e, 'winerror', 0) in (10053, 10054, 10058, 10038):
+                            self._abort(f"Socket Dead: {e}")
+                            break
+                        self._abort(f"Write Error: {e}")
+                        break
+
+                if total_sent == frame_len:
+                    self.metrics["msgs_out"] += 1
+
+            except Exception as e:
+                Logger.error(f"[{self.id}] Pump Fracture: {e}")
+                time.sleep(0.1)
+
+    def _abort(self, reason: str):
+        if self._is_alive:
+            self._is_alive = False
+            self.close()
+
+    def close(self):
+        """[ASCENSION 12]: IDEMPOTENT CLOSURE"""
+        self._is_alive = False
+        with self._write_lock:
+            self._high_queue.clear()
+            self._low_queue.clear()
+            self._queue_byte_size = 0
+        try:
+            self.sock.shutdown(socket.SHUT_RDWR)
+        except:
+            pass
+        try:
+            self.sock.close()
+        except:
+            pass

velm/core/daemon/transporter/physics.py

@@ -0,0 +1,51 @@
+# Path: scaffold/core/daemon/transporter/physics.py
+# -------------------------------------------------
+# LIF: INFINITY
+# =================================================================================
+# == THE MECHANIC (KERNEL TUNER - SHIELDED)                                      ==
+# =================================================================================
+
+import socket
+import logging
+from .constants import KERNEL_BUFFER_SIZE
+
+Logger = logging.getLogger("SocketPhysics")
+
+class SocketPhysics:
+    """
+    [THE SPEED OF LIGHT RITE]
+    Applies low-level kernel optimizations to the raw socket file descriptor.
+    Ensures the OS TCP stack is tuned for high-frequency RPC bursts.
+    """
+
+    @staticmethod
+    def tune(sock: socket.socket):
+        try:
+            # 1. ANNIHILATE NAGLE'S ALGORITHM
+            # We want packets sent NOW, not batched.
+            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+
+            # 2. EXPAND THE LUNGS (Kernel Buffers)
+            # Increase Kernel Send/Recv Buffers to prevent syscall blocking.
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, KERNEL_BUFFER_SIZE)
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, KERNEL_BUFFER_SIZE)
+
+            # 3. DETECT GHOSTS (Keep-Alive)
+            # Detect severed links from crashed clients immediately.
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+
+            # 4. PLATFORM SPECIFICS (Linux/Mac Tuning)
+            if hasattr(socket, 'TCP_KEEPIDLE'):
+                sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 60)
+            if hasattr(socket, 'TCP_KEEPINTVL'):
+                sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 10)
+            if hasattr(socket, 'TCP_KEEPCNT'):
+                sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 3)
+
+            # [ASCENSION]: Quick-Close
+            # Allow address reuse immediately after close (prevents ADDRINUSE)
+            sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+
+        except Exception as e:
+            # Physics failure is non-critical, we proceed with standard laws.
+            Logger.warning(f"Physics Tuning Minor Fracture: {e}")

velm/core/fusion/__init__.py

@@ -0,0 +1,6 @@
+# Path: scaffold/core/fusion/__init__.py
+# --------------------------------------
+
+from .engine import rust_fusion as rust
+
+__all__ = ["rust"]