PyMkDB 0.1.0__py3-none-any.whl

src/db/maintenance/task_scheduler.py

@@ -0,0 +1,73 @@
+"""
+TaskScheduler — periodic daemon thread that checks segments and enqueues
+compaction tasks via WriteQueue.
+"""
+
+import logging
+import threading
+import time
+
+logger = logging.getLogger(__name__)
+
+
+class TaskScheduler:
+    def __init__(self, store, interval: float = None):
+        """
+        Parameters
+        ----------
+        store    : src.db.objects.store.store
+        interval : check interval in seconds (default from file_config or 60)
+        """
+        self._store = store
+        if interval is None:
+            interval = float(getattr(store.config.file_config, "compaction_interval", 60))
+        self.interval = interval
+        self._stop_event = threading.Event()
+        self._thread = threading.Thread(
+            target=self._loop, daemon=True, name=f"TaskScheduler-{store.config.name}"
+        )
+
+    def start(self) -> None:
+        self._thread.start()
+
+    def stop(self) -> None:
+        self._stop_event.set()
+
+    def trigger_now(self, task_type: str = "compact") -> None:
+        """Immediately enqueue a compaction task for all eligible segments."""
+        self._run_checks(task_type)
+
+    # ------------------------------------------------------------------
+    # Internal loop
+    # ------------------------------------------------------------------
+
+    def _loop(self) -> None:
+        while not self._stop_event.wait(timeout=self.interval):
+            self._run_checks("compact")
+
+    def _run_checks(self, task_type: str) -> None:
+        store = self._store
+        if store.log_manager is None or store._write_queue is None:
+            return
+        from src.db.maintenance.compactor import Compactor
+        compactor = Compactor(store)
+
+        segments = store.log_manager.list_segments()
+        # Skip the active segment
+        active = store.log_manager.active_segment
+        for seq_int in segments:
+            if seq_int == active:
+                continue
+            seq_str = f"{seq_int:03d}"
+            try:
+                if task_type == "compact" and compactor.needs_compaction(seq_str):
+                    logger.info("TaskScheduler: enqueuing compact for segment %s", seq_str)
+                    store._write_queue.enqueue({
+                        "op":        "compact",
+                        "store":     store.config.name,
+                        "record_id": None,
+                        "delta":     {"segment": seq_int},
+                        "ts":        time.time(),
+                    })
+            except Exception as exc:
+                logger.warning("TaskScheduler: error checking segment %s: %s", seq_str, exc)

src/db/objects/store.py

@@ -0,0 +1,283 @@
+import os
+
+from src.db import mkdb
+from src.config.db import store_config as _store_config
+from src.db.query_workers import QueryDispatcher
+from src.db.storage.log_manager import LogManager
+from src.db.storage.index_manager import IndexManager
+from src.db.storage import blob_store
+from src.db.storage import serializer as _serializer
+from src.db.cache.ram_cache import RamCache
+from src.db.cache.write_queue import WriteQueue
+from src.db.query.query_engine import QueryEngine, QuerySyntaxError
+from src.db.maintenance.compactor import Compactor
+from src.db.maintenance.task_scheduler import TaskScheduler
+from src.db.parity.parity_manager import ParityManager
+
+
+class store:
+    def __init__(self, db: mkdb, store_config: _store_config):
+        self.db = db
+        self.config = store_config
+        self._dispatcher: QueryDispatcher | None = None
+        self.log_manager: LogManager | None = None
+        self.index_manager: IndexManager | None = None
+        self._ram_cache: RamCache | None = None
+        self._write_queue: WriteQueue | None = None
+        self.query_engine: QueryEngine | None = None
+        self._compactor: Compactor | None = None
+        self._task_scheduler: TaskScheduler | None = None
+        self._parity_manager: ParityManager | None = None
+
+    @property
+    def store_path(self):
+        return os.path.join(self.db.file_path, "stores", self.config.name)
+
+    @property
+    def dispatcher(self) -> QueryDispatcher:
+        """Lazy-initialised query dispatcher for this store."""
+        if self._dispatcher is None:
+            self._dispatcher = QueryDispatcher(
+                store_name=self.config.name,
+                base_path=self.store_path,
+                config=self.config.query_worker_config,
+            )
+            self._dispatcher.start()
+        return self._dispatcher
+
+    def setup(self):
+        """Setup the store based on the configuration."""
+        os.makedirs(self.store_path, exist_ok=True)
+        seg_threshold = getattr(self.config.file_config, "segment_threshold",
+                                self.config.file_config.file_size_split_trigger)
+        self.log_manager = LogManager(
+            store_path=self.store_path,
+            service=self.config.name,
+            segment_threshold=seg_threshold,
+        )
+        self.index_manager = IndexManager(
+            store_path=self.store_path,
+            service=self.config.name,
+        )
+        print(f"Store '{self.config.name}' initialized at {self.store_path}")
+        from src.server.event_log import emit as _emit
+        _emit("info", f"store:{self.config.name}", f"Store loaded from {self.store_path}")
+
+        # Parity manager — boot verification runs at startup
+        nsym = getattr(self.config.file_config, "parity_nsym", 10)
+        self._parity_manager = ParityManager(
+            store_path=self.store_path,
+            service=self.config.name,
+            nsym=nsym,
+        )
+        self._parity_manager.boot_verify_all(self.log_manager)
+
+        # RAM cache
+        self._ram_cache = RamCache(
+            max_size=self.config.ram_config.max_size,
+            ttl=self.config.ram_config.ttl,
+            clean_type=self.config.ram_config.clean_type,
+        )
+
+        # Write queue
+        debounce = getattr(self.config, "write_queue_config",
+                           None)
+        debounce_window = debounce.debounce_window if debounce else 5.0
+        max_pending     = debounce.max_pending     if debounce else 10_000
+        self._write_queue = WriteQueue(
+            debounce_window=debounce_window,
+            max_pending=max_pending,
+        )
+
+        def _handle_write(task: dict) -> None:
+            """Flush a debounced write task to the log/index layer."""
+            rid   = task["record_id"]
+            delta = task["delta"]
+            # Capture the active segment before write so we can detect rollover
+            seg_before = self.log_manager.active_segment if self.log_manager else None
+            # Call the low-level write (defined by WS-1); it writes to log + index
+            self._write_to_storage(rid, delta)
+            if self.query_engine is not None:
+                self.query_engine.on_write(rid, delta)
+            self.invalidate_cache(rid)
+            # On segment rollover: seal the old segment with parity + flush indexes
+            if self.log_manager and seg_before is not None and self.log_manager.active_segment != seg_before:
+                if self.query_engine is not None:
+                    self.query_engine.save_all()
+                if self._parity_manager is not None:
+                    try:
+                        self._parity_manager.generate(seg_before)
+                    except Exception:
+                        pass  # parity failure must never block writes
+
+        self._write_queue.register_handler("write", _handle_write)
+        self._write_queue.start()
+
+        # Query engine
+        self.query_engine = QueryEngine(self)
+        self.query_engine.build_indexes()
+
+        # Compaction
+        self._compactor = Compactor(self)
+
+        # Register compact handler on write queue
+        def _handle_compact(task: dict) -> None:
+            seq_int = task.get("delta", {}).get("segment")
+            if seq_int is not None:
+                self._compactor.compact_segment(int(seq_int))
+                if self._parity_manager is not None:
+                    try:
+                        self._parity_manager.generate(int(seq_int))
+                    except ImportError:
+                        pass   # reedsolo not installed; silently skip
+
+        self._write_queue.register_handler("compact", _handle_compact)
+
+        # Task scheduler
+        self._task_scheduler = TaskScheduler(self)
+        self._task_scheduler.start()
+
+    def query(self, operation: str, params: dict, timeout: float | None = None):
+        """
+        Submit a query task to the worker pool and return the result.
+
+        Parameters
+        ----------
+        operation : str   "read" | "query" | "count" | "exists" | "multi_read"
+        params    : dict  Operation-specific parameters.
+        timeout   : float Optional per-call timeout override.
+        """
+        return self.dispatcher.submit(operation, params, timeout=timeout)
+
+    def invalidate_cache(self, record_id: str) -> None:
+        """
+        Broadcast a cache-invalidation for record_id to all query workers.
+        Called by the write queue after a record is flushed to disk.
+        """
+        if self._dispatcher is not None:
+            self._dispatcher.invalidate(record_id)
+
+    def stop(self) -> None:
+        """Gracefully stop the query worker pool for this store."""
+        if self._dispatcher is not None:
+            self._dispatcher.stop()
+            self._dispatcher = None
+
+    def generate_id(self) -> str:
+        """Generate a unique record ID using this store's entity_config.
+
+        Samples random characters from ``entity_config.token_chars`` at
+        ``entity_config.token_length``.  Collision-checks against the live
+        index.  If every attempt at the current length fails AND
+        ``entity_config.auto_expand`` is True, the token_length is incremented
+        by 1, the config is saved, and generation retries at the new length.
+        """
+        import random
+        ec = self.config.entity_config
+        chars = ec.token_chars or "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+        max_attempts_per_length = 200
+
+        while True:
+            length = ec.token_length
+            for _ in range(max_attempts_per_length):
+                candidate = "".join(random.choices(chars, k=length))
+                # Accept immediately if the index says it doesn't exist
+                if self.index_manager is None or self.index_manager.get(candidate) is None:
+                    return candidate
+
+            # Exhausted attempts — keyspace is likely saturated at this length
+            if ec.auto_expand:
+                ec.token_length += 1
+                self.db.config.save()
+                # Loop again with the expanded length
+            else:
+                raise RuntimeError(
+                    f"Could not generate a unique ID for store '{self.config.name}' "
+                    f"after {max_attempts_per_length} attempts at token_length={length}. "
+                    "Enable auto_expand in the store's entity config, or increase token_length."
+                )
+
+    def write(self, record_id: str, flat_dict: dict) -> None:
+        """Write a record — updates cache immediately, flushes to disk via queue."""
+        if self._ram_cache is not None:
+            self._ram_cache.apply_delta(record_id, flat_dict)
+        if self._write_queue is not None:
+            self._write_queue.enqueue({
+                "op":        "write",
+                "store":     self.config.name,
+                "record_id": record_id,
+                "delta":     flat_dict,
+                "ts":        __import__("time").time(),
+            })
+        else:
+            # Fallback: direct write if queue not initialised
+            self._write_to_storage(record_id, flat_dict)
+
+    def _write_to_storage(self, record_id: str, flat_dict: dict) -> None:
+        """Write directly to log + index (bypasses cache and queue)."""
+        if self.log_manager is None or self.index_manager is None:
+            raise RuntimeError("Store is not set up. Call setup() first.")
+        blob_threshold = getattr(self.config.file_config, "blob_threshold", 5 * 1024 * 1024)
+        from src.db.storage import serializer as _serializer
+        from src.db.storage import blob_store
+        line_str = _serializer.serialize_record(record_id, flat_dict)
+        if len(line_str.encode("utf-8")) > blob_threshold:
+            seg, offset, size = blob_store.write_blob(self.store_path, record_id, line_str)
+        else:
+            seg, offset, size = self.log_manager.append(record_id, line_str)
+        self.index_manager.set(record_id, seg, offset, size)
+
+    def read(self, record_id: str) -> dict | None:
+        """Read a record — checks RAM cache first, falls back to disk."""
+        if self._ram_cache is not None:
+            cached = self._ram_cache.get(record_id)
+            if cached is not None:
+                return cached
+        if self.index_manager is None:
+            raise RuntimeError("Store is not set up. Call setup() first.")
+        entry = self.index_manager.get(record_id)
+        if entry is None:
+            return None
+        seg, offset, size = entry
+        if self.log_manager is None:
+            raise RuntimeError("Store is not set up. Call setup() first.")
+        from src.db.storage import serializer as _serializer
+        line_str = self.log_manager.read(seg, offset, size)
+        _, flat_dict = _serializer.deserialize_record(line_str)
+        if self._ram_cache is not None:
+            self._ram_cache.set(record_id, flat_dict)
+        return flat_dict
+
+    def delete(self, record_id: str) -> None:
+        """Soft-delete a record."""
+        if self._ram_cache is not None:
+            self._ram_cache.delete(record_id)
+        if self.index_manager is None:
+            raise RuntimeError("Store is not set up. Call setup() first.")
+        self.index_manager.delete(record_id)
+        # Update query indexes
+        if self.query_engine is not None:
+            # Read old values for index removal — if not in cache, skip (index will be stale until rebuild)
+            old = self._ram_cache.get(record_id) if self._ram_cache else None
+            if old:
+                self.query_engine.on_delete(record_id, old)
+
+    def teardown(self) -> None:
+        """Flush and close storage handles. Called during server shutdown."""
+        from src.server.event_log import emit as _emit
+        _emit("info", f"store:{self.config.name}", "Store shutting down — flushing writes")
+        if self._task_scheduler is not None:
+            self._task_scheduler.stop()
+        # Flush the write queue FIRST so all pending writes hit the log and
+        # update the query engine's in-memory indexes before we save them.
+        if self._write_queue is not None:
+            self._write_queue.flush_all()
+            self._write_queue.stop()
+        if self.query_engine is not None:
+            self.query_engine.save_all()
+            print(f"[{self.config.name}] Query indexes saved.")
+        if self.index_manager is not None and self.index_manager._dirty:
+            self.index_manager.save_full()
+        if self.log_manager is not None:
+            self.log_manager.close()
+        self.stop()  # stops query dispatcher

src/db/parity/parity_manager.py

@@ -0,0 +1,196 @@
+"""
+ParityManager — Reed-Solomon parity generation and verification for log segments.
+
+Parity files: {store_path}/{service}_{NNN}.parity
+These files contain the Reed-Solomon parity symbols for the corresponding .log segment.
+
+External dependency: reedsolo >= 1.7  (pip install reedsolo)
+Imports are deferred to operation methods so the module loads cleanly even
+if reedsolo is not installed; a clear ImportError with install instructions
+is raised only when a parity operation is invoked.
+"""
+
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+_REEDSOLO_MISSING = (
+    "reedsolo is required for parity operations. "
+    "Install it with: pip install reedsolo"
+)
+
+
+class ParityManager:
+    def __init__(self, store_path: str, service: str, nsym: int = 10):
+        """
+        Parameters
+        ----------
+        store_path : absolute path to the store directory
+        service    : store name (file prefix)
+        nsym       : Reed-Solomon parity symbol count (default 10)
+        """
+        self.store_path = store_path
+        self.service    = service
+        self.nsym       = nsym
+
+    # ------------------------------------------------------------------
+    # Path helpers
+    # ------------------------------------------------------------------
+
+    def _log_path(self, seq_int: int) -> str:
+        return os.path.join(self.store_path, f"{self.service}_{seq_int:03d}.log")
+
+    def _parity_path(self, seq_int: int) -> str:
+        return os.path.join(self.store_path, f"{self.service}_{seq_int:03d}.parity")
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def generate(self, seq_int: int) -> None:
+        """
+        Read the .log segment, encode via Reed-Solomon, write parity to .parity file.
+        Overwrites any existing .parity file.
+
+        Parity file format: concatenated ECC symbols for each chunk
+        (nsym bytes per chunk of chunk_size = nsize-nsym data bytes).
+        This avoids the single-chunk limitation of GF(2^8) (max 255 bytes)
+        and keeps parity files small (~nsym/chunk_size ≈ 4% of log size).
+        """
+        try:
+            import reedsolo
+        except ImportError:
+            raise ImportError(_REEDSOLO_MISSING)
+
+        log_path = self._log_path(seq_int)
+        if not os.path.exists(log_path):
+            logger.warning("generate: log segment %03d not found, skipping", seq_int)
+            return
+
+        with open(log_path, "rb") as fh:
+            data = fh.read()
+
+        codec = reedsolo.RSCodec(self.nsym)
+        chunk_size = codec.nsize - codec.nsym  # data bytes per chunk (default 245)
+        ecc_parts = []
+        for i in range(0, len(data), chunk_size):
+            chunk = data[i:i + chunk_size]
+            enc_chunk = bytearray(codec.encode(chunk))
+            ecc_parts.append(enc_chunk[len(chunk):])  # only the nsym ECC bytes
+        parity_bytes = b"".join(ecc_parts)
+
+        parity_path = self._parity_path(seq_int)
+        tmp_path = parity_path + ".tmp"
+        with open(tmp_path, "wb") as fh:
+            fh.write(parity_bytes)
+        os.replace(tmp_path, parity_path)
+        logger.debug("generate: parity written for segment %03d (%d bytes)", seq_int, len(parity_bytes))
+
+    def verify_and_repair(self, seq_int: int) -> bool:
+        """
+        Verify and optionally repair a log segment using its parity file.
+
+        Returns
+        -------
+        True  — segment is clean or was repaired successfully
+        False — segment is corrupt beyond repair
+        """
+        try:
+            import reedsolo
+        except ImportError:
+            raise ImportError(_REEDSOLO_MISSING)
+
+        log_path    = self._log_path(seq_int)
+        parity_path = self._parity_path(seq_int)
+
+        if not os.path.exists(log_path):
+            logger.warning("verify_and_repair: log segment %03d not found", seq_int)
+            return False
+        if not os.path.exists(parity_path):
+            logger.warning("verify_and_repair: parity file for segment %03d not found; generating now", seq_int)
+            self.generate(seq_int)
+            return True   # freshly generated == clean by definition
+
+        with open(log_path, "rb") as fh:
+            log_bytes = fh.read()
+        with open(parity_path, "rb") as fh:
+            parity_bytes = fh.read()
+
+        codec = reedsolo.RSCodec(self.nsym)
+        chunk_size = codec.nsize - codec.nsym  # data bytes per chunk (default 245)
+        repaired = bytearray()
+        ecc_offset = 0
+        try:
+            for i in range(0, len(log_bytes), chunk_size):
+                chunk = log_bytes[i:i + chunk_size]
+                ecc = parity_bytes[ecc_offset:ecc_offset + codec.nsym]
+                dec_tuple = codec.decode(bytes(chunk) + bytes(ecc))
+                repaired.extend(dec_tuple[0])
+                ecc_offset += codec.nsym
+        except reedsolo.ReedSolomonError as exc:
+            logger.error(
+                "verify_and_repair: segment %03d is corrupt beyond repair: %s",
+                seq_int, exc,
+            )
+            return False
+
+        decoded_bytes = bytes(repaired)
+        if decoded_bytes != log_bytes:
+            # Corruption detected but repairable
+            logger.warning(
+                "verify_and_repair: segment %03d had corruption — repaired and rewritten",
+                seq_int,
+            )
+            from src.server.event_log import emit as _emit
+            _emit("recovery", f"store:{self.service}",
+                  f"Segment {self.service}_{seq_int:03d}.log had corruption — repaired via Reed-Solomon")
+            tmp_path = log_path + ".tmp"
+            with open(tmp_path, "wb") as fh:
+                fh.write(decoded_bytes)
+            os.replace(tmp_path, log_path)
+
+        return True
+
+    def boot_verify_all(self, log_manager) -> None:
+        """
+        Called from store.setup() after LogManager is initialised.
+        Iterates all sealed segments and calls verify_and_repair on each.
+        The active (highest-numbered) segment is excluded from verification
+        because it is still being appended to — its parity is always regenerated
+        fresh instead so subsequent rollover-based parity generation is correct.
+        """
+        from src.server.event_log import emit as _emit
+        all_segs = log_manager.list_segments()
+        active   = log_manager.active_segment
+        for seq_int in all_segs:
+            if seq_int == active:
+                # Always regenerate parity for the active segment — it may have
+                # grown since the last time parity was written.
+                try:
+                    self.generate(seq_int)
+                    logger.debug("boot_verify_all: regenerated parity for active segment %03d", seq_int)
+                except ImportError:
+                    logger.info(
+                        "boot_verify_all: reedsolo not installed — parity skipped. "
+                        "Install with: pip install reedsolo"
+                    )
+                except Exception as exc:
+                    logger.warning("boot_verify_all: could not generate parity for active segment %03d: %s", seq_int, exc)
+                continue
+            try:
+                ok = self.verify_and_repair(seq_int)
+                if not ok:
+                    logger.error("boot_verify_all: segment %03d could not be repaired", seq_int)
+                    _emit("error", f"store:{self.service}",
+                          f"Segment {self.service}_{seq_int:03d}.log is corrupt beyond repair")
+            except ImportError:
+                logger.info(
+                    "boot_verify_all: reedsolo not installed — parity verification skipped. "
+                    "Install with: pip install reedsolo"
+                )
+                return   # skip all remaining segments
+            except Exception as exc:
+                logger.warning("boot_verify_all: error on segment %03d: %s", seq_int, exc)
+                _emit("warning", f"store:{self.service}",
+                      f"Parity check error on segment {self.service}_{seq_int:03d}.log: {exc}")

src/db/query/__init__.py

@@ -0,0 +1 @@
+