PyMkDB 0.1.0__py3-none-any.whl

src/db/__init__.py

@@ -0,0 +1,207 @@
+import os
+import time
+from typing import Union
+
+from colorama import Fore
+from src.config.db import mkdb_config
+from src.server.coms.http import HTTPServer
+
+
+class mkdb:
+    def __init__(self, config:Union[dict, mkdb_config]):
+        from src.db.objects.store import store
+
+        if type(config) == dict:
+            config = mkdb_config(config)
+        self.config:mkdb_config = config #type: ignore
+        self.stores:dict[str, "store"] = {}
+        self._servers: dict = {}   # "http" | "socket" | "control" -> instance
+        self._started_at: float = 0.0  # unix timestamp of when run() was called
+        
+        self.setup()
+
+    @property
+    def file_path(self):
+        return self.config.base_path
+
+    def __verify__(self, config:Union[mkdb_config, None]=None):
+        """
+        Verify the database configuration and setup.
+        """
+        if config is None:
+            config = self.config
+
+        if config.name in [None, ""]:
+            raise ValueError("Database name cannot be empty.")
+        if config.base_path in [None, ""]:
+            raise ValueError("Base path cannot be empty.")
+        if config.servers.socket_server.enabled:
+            if config.servers.socket_server.address.host in [None, ""]:
+                raise ValueError("Socket server enabled but no host specified.")
+            if config.servers.socket_server.address.port == 0:
+                raise ValueError("Socket server enabled but no port specified.")
+        if config.servers.http_server.enabled:
+            if config.servers.http_server.address.host in [None, ""]:
+                raise ValueError("HTTP server enabled but no host specified.")
+            if config.servers.http_server.address.port == 0:
+                raise ValueError("HTTP server enabled but no port specified.")
+    
+    def setup(self):
+        try:
+            self.__verify__()
+        except Exception as e:
+            print(f"Error verifying database configuration: {Fore.RED}{e}{Fore.RESET}")
+            raise e
+            return
+        
+        os.makedirs(os.path.join(self.file_path, "stores"), exist_ok=True)
+        for store_name, store_config in self.config.stores.items():
+            from src.db.objects.store import store
+            if store_name not in self.stores:
+                s = store(self, store_config)
+                self.stores[store_name] = s
+            else:
+                s = self.stores[store_name]
+                s.config.update(store_config.json)
+            s.setup()
+
+    def run(self):
+        print(f"{Fore.GREEN}Database '{self.config.name}' initialized at {os.getcwd()}{Fore.RESET}")
+
+        print("Starting servers...")
+        self._started_at = time.time()
+        if self.config.servers.http_server.enabled:
+            from src.server.coms.http import HTTPServer
+            from src.server.coms.http_handlers import HTTPDataHandler
+            HTTPDataHandler.database = self
+            self._servers["http"] = HTTPServer(
+                name="Communication-HTTP",
+                host=self.config.servers.http_server.address.host,
+                port=self.config.servers.http_server.address.port,
+                responder=HTTPDataHandler,
+            )
+
+        if self.config.servers.socket_server.enabled:
+            from src.server.coms.socket import SocketServer
+            _sock_srv = SocketServer(
+                host=self.config.servers.socket_server.address.host,
+                port=self.config.servers.socket_server.address.port,
+                database=self,
+                heartbeat_interval=self.config.servers.socket_server.heartbeat_interval,
+                max_clients=self.config.servers.socket_server.max_clients,
+                recv_timeout=self.config.servers.socket_server.recv_timeout,
+            )
+            _sock_srv.start()
+            self._servers["socket"] = _sock_srv
+
+        if self.config.servers.control_server.enabled:
+            from src.server.control.server import start_control_server
+            self._servers["control"] = start_control_server(
+                host=self.config.servers.control_server.address.host,
+                port=self.config.servers.control_server.address.port,
+                database=self
+            )
+            
+        print(f"MkDB {Fore.CYAN}'{self.config.name}'{Fore.RESET} is running at {Fore.GREEN}{os.getcwd()}{Fore.RESET}")
+
+        while True:
+            try:
+                time.sleep(1)
+                pass
+            except KeyboardInterrupt:
+                print(f"{Fore.YELLOW}Shutting down MkDB '{self.config.name}'...{Fore.RESET}")
+                break
+        self.shutdown()
+
+    def get_server_status(self) -> dict:
+        """Return running/stopped status for all three servers."""
+        return {name: (self._servers.get(name) is not None) for name in ("http", "socket", "control")}
+
+    def get_uptime(self) -> float:
+        """Return seconds since run() was called, or 0 if not yet started."""
+        return (time.time() - self._started_at) if self._started_at else 0.0
+
+    def stop_server(self, name: str) -> None:
+        """Stop a named server and clear its reference."""
+        srv = self._servers.get(name)
+        if srv is None:
+            return
+        if hasattr(srv, "stop"):
+            srv.stop()
+        elif hasattr(srv, "shutdown"):
+            try:
+                srv.shutdown()
+            except Exception:
+                pass
+        self._servers[name] = None
+
+    def start_server(self, name: str) -> None:
+        """Start a named server using current config. No-op if already running."""
+        if self._servers.get(name) is not None:
+            return
+        cfg = self.config.servers
+        if name == "http":
+            if not cfg.http_server.enabled:
+                return
+            from src.server.coms.http import HTTPServer
+            from src.server.coms.http_handlers import HTTPDataHandler
+            HTTPDataHandler.database = self
+            self._servers["http"] = HTTPServer(
+                name="Communication-HTTP",
+                host=cfg.http_server.address.host,
+                port=cfg.http_server.address.port,
+                responder=HTTPDataHandler,
+            )
+        elif name == "socket":
+            if not cfg.socket_server.enabled:
+                return
+            from src.server.coms.socket import SocketServer
+            srv = SocketServer(
+                host=cfg.socket_server.address.host,
+                port=cfg.socket_server.address.port,
+                database=self,
+                heartbeat_interval=cfg.socket_server.heartbeat_interval,
+                max_clients=cfg.socket_server.max_clients,
+                recv_timeout=cfg.socket_server.recv_timeout,
+            )
+            srv.start()
+            self._servers["socket"] = srv
+        elif name == "control":
+            if not cfg.control_server.enabled:
+                return
+            from src.server.control.server import start_control_server
+            self._servers["control"] = start_control_server(
+                host=cfg.control_server.address.host,
+                port=cfg.control_server.address.port,
+                database=self,
+            )
+
+    def restart_server(self, name: str) -> None:
+        """Stop then start a named server."""
+        self.stop_server(name)
+        self.start_server(name)
+
+    def shutdown(self) -> None:
+        """Stop all servers and tear down all stores."""
+        for name in ("http", "socket", "control"):
+            self.stop_server(name)
+        for s in self.stores.values():
+            try:
+                s.teardown()
+            except Exception:
+                pass
+        self.stores.clear()
+        print(f"{Fore.YELLOW}MkDB '{self.config.name}' shut down.{Fore.RESET}")
+
+    def update_from_config(self, new_config:Union[dict, mkdb_config]):
+        if type(new_config) == dict:
+            new_config = mkdb_config(new_config)
+        try:
+            self.__verify__(new_config) #type: ignore
+            self.config = new_config #type: ignore
+            print(f"{Fore.GREEN}Configuration updated successfully.{Fore.RESET}")
+        except Exception as e:
+            print(f"Error updating database configuration: {Fore.RED}{e}{Fore.RESET}")
+            return
+        
+        self.setup()

src/db/cache/__init__.py

@@ -0,0 +1 @@
+

src/db/cache/ram_cache.py

@@ -0,0 +1,144 @@
+"""
+RamCache — in-process record cache for one store.
+
+Supports three eviction strategies controlled by ram_config.clean_type:
+  "lru" — evict least-recently-used entry
+  "lfu" — evict least-frequently-used entry
+  "ttl" — evict only entries whose TTL has expired (no count-based eviction)
+
+All public methods are thread-safe via a single threading.Lock.
+"""
+
+import threading
+import time
+from collections import OrderedDict
+
+
+class RamCache:
+    def __init__(self, max_size: int, ttl: int, clean_type: str = "lru"):
+        self.max_size   = max_size
+        self.ttl        = ttl           # seconds; 0 = never expire
+        self.clean_type = clean_type
+
+        self._store:       dict          = {}             # record_id -> flat dict
+        self._access_log:  OrderedDict   = OrderedDict()  # record_id -> None (LRU order)
+        self._freq:        dict          = {}             # record_id -> access count (LFU)
+        self._timestamps:  dict          = {}             # record_id -> last_access float
+        self._lock:        threading.Lock = threading.Lock()
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def get(self, record_id: str):
+        """Return the cached flat dict, or None on miss / expired entry."""
+        with self._lock:
+            if record_id not in self._store:
+                return None
+            if self._is_expired(record_id):
+                self._evict_one(record_id)
+                return None
+            self._touch(record_id)
+            return dict(self._store[record_id])   # return a shallow copy
+
+    def set(self, record_id: str, flat_dict: dict) -> None:
+        """Insert or overwrite an entry, evicting if over max_size."""
+        with self._lock:
+            if record_id in self._store:
+                self._store[record_id] = dict(flat_dict)
+                self._touch(record_id)
+            else:
+                self._store[record_id] = dict(flat_dict)
+                self._access_log[record_id] = None
+                self._freq[record_id] = 1
+                self._timestamps[record_id] = time.time()
+                while len(self._store) > self.max_size:
+                    self.evict()
+
+    def delete(self, record_id: str) -> None:
+        """Remove an entry from the cache."""
+        with self._lock:
+            self._evict_one(record_id)
+
+    def size(self) -> int:
+        """Return the current number of cached entries."""
+        with self._lock:
+            return len(self._store)
+
+    def estimated_bytes(self) -> int:
+        """
+        Rough estimate of memory used by cached data.
+        Counts the total character length of all cached flat-dict values.
+        Accurate enough for dashboard display; not a true malloc measurement.
+        """
+        import sys
+        with self._lock:
+            total = 0
+            for record in self._store.values():
+                for k, v in record.items():
+                    total += sys.getsizeof(k) + sys.getsizeof(v)
+            total += sys.getsizeof(self._store)
+            return total
+
+    def apply_delta(self, record_id: str, delta: dict) -> None:
+        """
+        Merge delta flat-paths into the existing cached object (upsert).
+        If the record is not cached, create a new entry with just the delta.
+        """
+        with self._lock:
+            if record_id in self._store:
+                self._store[record_id].update(delta)
+                self._touch(record_id)
+            else:
+                self._store[record_id] = dict(delta)
+                self._access_log[record_id] = None
+                self._freq[record_id] = 1
+                self._timestamps[record_id] = time.time()
+
+    def evict(self) -> None:
+        """
+        Evict one entry according to the configured strategy.
+        Called with _lock already held.
+        """
+        if not self._store:
+            return
+        if self.clean_type == "lru":
+            oldest_id = next(iter(self._access_log))
+            self._evict_one(oldest_id)
+        elif self.clean_type == "lfu":
+            lfu_id = min(self._freq, key=lambda k: self._freq[k])
+            self._evict_one(lfu_id)
+        elif self.clean_type == "ttl":
+            # Evict all expired entries
+            now = time.time()
+            expired = [rid for rid, ts in self._timestamps.items()
+                       if self.ttl > 0 and (now - ts) > self.ttl]
+            for rid in expired:
+                self._evict_one(rid)
+            # If nothing expired but we're still over max_size, fall back to LRU
+            if len(self._store) > self.max_size:
+                oldest_id = next(iter(self._access_log))
+                self._evict_one(oldest_id)
+
+    # ------------------------------------------------------------------
+    # Internal helpers (all called with _lock held)
+    # ------------------------------------------------------------------
+
+    def _touch(self, record_id: str) -> None:
+        """Update LRU order, frequency, and timestamp for an accessed entry."""
+        self._access_log.move_to_end(record_id)
+        self._freq[record_id] = self._freq.get(record_id, 0) + 1
+        self._timestamps[record_id] = time.time()
+
+    def _is_expired(self, record_id: str) -> bool:
+        if self.ttl <= 0:
+            return False
+        ts = self._timestamps.get(record_id, 0)
+        return (time.time() - ts) > self.ttl
+
+    def _evict_one(self, record_id: str) -> None:
+        """Remove a single record_id from all internal structures."""
+        self._store.pop(record_id, None)
+        self._access_log.pop(record_id, None)
+        self._freq.pop(record_id, None)
+        self._timestamps.pop(record_id, None)

src/db/cache/write_queue.py

@@ -0,0 +1,156 @@
+"""
+WriteQueue — debounced async write queue for one store.
+
+Enqueued "write" operations are buffered and coalesced per record_id for
+debounce_window seconds before being flushed to the registered handler.
+Operations with op == "compact" or "rebuild_index" bypass debouncing and
+are pushed directly to the queue.
+
+A dead-letter list captures tasks whose handler raised an exception, capped
+at 1000 entries (oldest dropped first).
+"""
+
+import logging
+import queue
+import threading
+import time
+from typing import Callable
+
+logger = logging.getLogger(__name__)
+
+WORKER_SENTINEL = object()
+
+
+class WriteQueue:
+    def __init__(self, debounce_window: float = 5.0, max_pending: int = 10_000):
+        self.debounce_window = debounce_window
+        self.max_pending     = max_pending
+
+        self._queue:         queue.Queue = queue.Queue()
+        self._pending:       dict        = {}    # record_id -> accumulated delta dict
+        self._timers:        dict        = {}    # record_id -> enqueue timestamp
+        self._handlers:      dict        = {}    # op_str -> Callable
+        self._dead_letter:   list        = []    # failed tasks, capped at 1000
+        self._stop_event:    threading.Event = threading.Event()
+        self._worker_thread: threading.Thread = threading.Thread(
+            target=self._worker_loop, daemon=True, name="WriteQueue-worker"
+        )
+        self._lock: threading.Lock = threading.Lock()
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def start(self) -> None:
+        """Start the background worker thread."""
+        self._worker_thread.start()
+
+    def register_handler(self, op: str, fn: Callable) -> None:
+        """Map an operation string to a handler function."""
+        self._handlers[op] = fn
+
+    def enqueue(self, task: dict) -> None:
+        """
+        Add a task to the queue.
+
+        For "write" ops: coalesce delta into _pending[record_id] and reset timer.
+        For all other ops: push directly to _queue.
+
+        Raises RuntimeError if _pending is at max_pending capacity.
+        """
+        op = task.get("op")
+        if op == "write":
+            record_id = task.get("record_id")
+            delta     = task.get("delta", {})
+            if record_id is None:
+                logger.warning("WriteQueue: write task missing record_id, skipping")
+                return
+            with self._lock:
+                if record_id not in self._pending and len(self._pending) >= self.max_pending:
+                    raise RuntimeError(
+                        f"WriteQueue pending buffer full ({self.max_pending} entries)"
+                    )
+                if record_id in self._pending:
+                    self._pending[record_id].update(delta)
+                else:
+                    self._pending[record_id] = dict(delta)
+                self._timers[record_id] = time.time()
+        else:
+            self._queue.put(task)
+
+    def flush_all(self) -> None:
+        """Force-drain all pending write timers immediately (call before stop)."""
+        with self._lock:
+            for record_id, delta in list(self._pending.items()):
+                store = self._timers.get(record_id)   # reuse timer slot for store name
+                # Build a flush task from whatever is in _pending
+                task = {
+                    "op":        "write",
+                    "record_id": record_id,
+                    "delta":     delta,
+                    "ts":        time.time(),
+                }
+                self._queue.put(task)
+            self._pending.clear()
+            self._timers.clear()
+
+    def stop(self) -> None:
+        """Signal stop, flush, and join the worker thread (10 s timeout)."""
+        self.flush_all()
+        self._stop_event.set()
+        self._queue.put(WORKER_SENTINEL)
+        self._worker_thread.join(timeout=10)
+
+    def get_dead_letters(self) -> list:
+        """Return a copy of the dead-letter list for admin inspection."""
+        return list(self._dead_letter)
+
+    # ------------------------------------------------------------------
+    # Worker loop (runs in daemon thread)
+    # ------------------------------------------------------------------
+
+    def _worker_loop(self) -> None:
+        while not self._stop_event.is_set():
+            # 1. Drain expired pending entries
+            now = time.time()
+            with self._lock:
+                due = [
+                    rid for rid, ts in self._timers.items()
+                    if (now - ts) >= self.debounce_window
+                ]
+                for record_id in due:
+                    task = {
+                        "op":        "write",
+                        "record_id": record_id,
+                        "delta":     self._pending.pop(record_id),
+                        "ts":        self._timers.pop(record_id),
+                    }
+                    self._queue.put(task)
+
+            # 2. Pull and dispatch one item
+            try:
+                task = self._queue.get(timeout=0.1)
+            except queue.Empty:
+                continue
+
+            if task is WORKER_SENTINEL:
+                break
+
+            self._dispatch(task)
+
+    def _dispatch(self, task: dict) -> None:
+        op = task.get("op")
+        handler = self._handlers.get(op)
+        if handler is None:
+            logger.warning("WriteQueue: no handler for op=%r", op)
+            return
+        try:
+            handler(task)
+        except Exception as exc:
+            logger.error(
+                "WriteQueue: handler for op=%r record_id=%r raised: %s",
+                op, task.get("record_id"), exc,
+            )
+            self._dead_letter.append(task)
+            if len(self._dead_letter) > 1000:
+                self._dead_letter.pop(0)

src/db/maintenance/compactor.py

@@ -0,0 +1,118 @@
+"""
+Compactor — rewrites log segments to reclaim space from deleted/overwritten records.
+
+For each segment:
+1. Collect all active record IDs whose current segment == this segment.
+2. Re-read each record line and write it to a .tmp file, tracking new offsets.
+3. Atomically replace the .log with the .tmp via os.replace.
+4. Batch-update IndexManager with new offsets and call save_full().
+"""
+
+import logging
+import os
+
+logger = logging.getLogger(__name__)
+
+
+class Compactor:
+    def __init__(self, store):
+        """
+        Parameters
+        ----------
+        store : src.db.objects.store.store
+        """
+        self._store = store
+
+    # ------------------------------------------------------------------
+    # Size helpers
+    # ------------------------------------------------------------------
+
+    def compute_logical_size(self, seq_str: str) -> int:
+        """Sum of all stored entry sizes in index_manager._map for this segment."""
+        idx = self._store.index_manager
+        if idx is None:
+            return 0
+        return sum(
+            size
+            for _rid, (seg, _off, size) in idx._map.items()
+            if seg == seq_str
+        )
+
+    def needs_compaction(self, seq_str: str, dead_ratio_threshold: float = None) -> bool:
+        """
+        Return True if the dead-record ratio exceeds the threshold.
+        dead_ratio = (physical_size - logical_size) / physical_size
+        """
+        store = self._store
+        if store.log_manager is None:
+            return False
+        # Default threshold from config
+        if dead_ratio_threshold is None:
+            dead_ratio_threshold = getattr(
+                store.config.file_config, "compaction_dead_ratio", 0.3
+            )
+        seq_int = int(seq_str)
+        path = store.log_manager.segment_path(seq_int)
+        if not os.path.exists(path):
+            return False
+        physical_size = os.path.getsize(path)
+        if physical_size == 0:
+            return False
+        logical_size = self.compute_logical_size(seq_str)
+        ratio = (physical_size - logical_size) / physical_size
+        return ratio > dead_ratio_threshold
+
+    # ------------------------------------------------------------------
+    # Compact
+    # ------------------------------------------------------------------
+
+    def compact_segment(self, seq_int: int) -> None:
+        """
+        Compact one log segment in place.
+        Skips the active (currently-written) segment to avoid corruption.
+        """
+        store = self._store
+        if store.log_manager is None or store.index_manager is None:
+            raise RuntimeError("Store storage not initialised.")
+
+        # Never compact the active segment (it has an open file handle)
+        if seq_int == store.log_manager.active_segment:
+            logger.info("compact_segment: skipping active segment %03d", seq_int)
+            return
+
+        seq_str = f"{seq_int:03d}"
+        active_ids = store.index_manager.records_in_segment(seq_str)
+        if not active_ids:
+            logger.info("compact_segment: segment %s has no live records, skipping", seq_str)
+            return
+
+        seg_path = store.log_manager.segment_path(seq_int)
+        tmp_path = seg_path + ".tmp"
+
+        new_offsets: dict = {}   # record_id -> (seq_str, new_offset, new_size)
+
+        with open(tmp_path, "w", encoding="utf-8") as out_fh:
+            for record_id in active_ids:
+                entry = store.index_manager.get(record_id)
+                if entry is None:
+                    continue
+                seg, offset, size = entry
+                try:
+                    line_str = store.log_manager.read(seg, offset, size)
+                except Exception as exc:
+                    logger.warning("compact_segment: could not read %s: %s", record_id, exc)
+                    continue
+                # Write to .tmp
+                new_offset = out_fh.tell()
+                out_fh.write(line_str + "\n")
+                new_size = len((line_str + "\n").encode("utf-8"))
+                new_offsets[record_id] = (seq_str, new_offset, new_size)
+
+        # Atomic swap
+        os.replace(tmp_path, seg_path)
+        logger.info("compact_segment: segment %s compacted (%d records kept)", seq_str, len(new_offsets))
+
+        # Update index
+        for record_id, (seg, new_offset, new_size) in new_offsets.items():
+            store.index_manager._map[record_id] = (seg, new_offset, new_size)
+        store.index_manager.save_full()