PyMkDB 0.1.0__py3-none-any.whl

src/db/query_workers/worker.py

@@ -0,0 +1,292 @@
+"""
+QueryWorker — runs inside a worker process (or thread) to resolve QueryTasks.
+
+Each worker owns:
+  - A lightweight LRU + TTL RAM cache keyed by record_id.
+  - Read-only access to the store's disk files via the storage layer.
+  - A private invalidation queue so the dispatcher can evict stale entries.
+
+Entry points
+------------
+worker_process_main  — for multiprocessing.Process targets
+worker_thread_main   — for threading.Thread targets (single-worker / no-GIL mode)
+
+Both share the same _worker_loop implementation; the only difference is how
+sys.path is initialised (process needs it, thread already has it).
+"""
+
+import os
+import sys
+import time
+import queue
+import logging
+from collections import OrderedDict
+from typing import Any, Optional
+
+# The sentinel object placed in the work queue to signal graceful shutdown.
+WORKER_SENTINEL = None
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Minimal worker-local LRU + TTL cache
+# (A full RamCache from WS-2 will replace this once implemented.)
+# ---------------------------------------------------------------------------
+
+class _WorkerCache:
+    """
+    Lightweight LRU cache with per-entry TTL.
+
+    Operations are O(1) (dict + OrderedDict).
+    """
+
+    def __init__(self, max_size: int, ttl: float):
+        self._max  = max(1, max_size)
+        self._ttl  = float(ttl)
+        self._data: dict[str, Any]          = {}
+        self._order: OrderedDict[str, None] = OrderedDict()
+        self._ts:   dict[str, float]        = {}
+
+    # -- public API ----------------------------------------------------------
+
+    def get(self, record_id: str) -> Optional[Any]:
+        if record_id not in self._data:
+            return None
+        if time.monotonic() - self._ts[record_id] > self._ttl:
+            self._evict_one(record_id)
+            return None
+        self._order.move_to_end(record_id)
+        return self._data[record_id]
+
+    def set(self, record_id: str, value: Any) -> None:
+        if record_id in self._data:
+            self._order.move_to_end(record_id)
+        else:
+            if len(self._data) >= self._max:
+                oldest, _ = self._order.popitem(last=False)
+                del self._data[oldest]
+                del self._ts[oldest]
+            self._order[record_id] = None
+        self._data[record_id] = value
+        self._ts[record_id]   = time.monotonic()
+
+    def delete(self, record_id: str) -> None:
+        self._evict_one(record_id)
+
+    def clear(self) -> None:
+        self._data.clear()
+        self._order.clear()
+        self._ts.clear()
+
+    def __len__(self) -> int:
+        return len(self._data)
+
+    # -- internal ------------------------------------------------------------
+
+    def _evict_one(self, record_id: str) -> None:
+        self._data.pop(record_id, None)
+        self._order.pop(record_id, None)
+        self._ts.pop(record_id, None)
+
+
+# ---------------------------------------------------------------------------
+# Task resolution — integration point for WS-1 and WS-3
+# ---------------------------------------------------------------------------
+
+def _resolve(task_dict: dict, cache: _WorkerCache, base_path: str) -> Any:
+    """
+    Dispatch a task dict to the correct resolver.
+
+    Integration notes
+    -----------------
+    WS-1 (storage layer):
+        Replace the NotImplementedError blocks in _read and _exists with:
+            index_mgr = IndexManager(base_path, store_name)
+            log_mgr   = LogManager(base_path, store_name)
+            raw_line  = log_mgr.read(*index_mgr.get(record_id))
+            # parse flat line -> dict
+
+    WS-3 (query engine):
+        Replace the NotImplementedError blocks in _query and _count with:
+            engine = QueryEngine(store)
+            return engine.query(filter_dict)
+    """
+    op         = task_dict["operation"]
+    store_name = task_dict["store_name"]
+    params     = task_dict["params"]
+
+    if op == "read":
+        return _read(store_name, params, cache, base_path)
+    if op == "multi_read":
+        return _multi_read(store_name, params, cache, base_path)
+    if op == "exists":
+        return _exists(store_name, params, cache, base_path)
+    if op == "query":
+        return _query(store_name, params, cache, base_path)
+    if op == "count":
+        return _count(store_name, params, cache, base_path)
+    raise ValueError(f"Unknown operation: {op!r}")
+
+
+def _read(store_name: str, params: dict, cache: _WorkerCache, base_path: str) -> dict:
+    record_id = params.get("record_id", "")
+    if not record_id:
+        raise ValueError("record_id required for 'read'")
+
+    cached = cache.get(record_id)
+    if cached is not None:
+        return cached
+
+    # TODO(WS-1): seek by byte offset from IndexManager / LogManager
+    # from src.db.storage.index_manager import IndexManager
+    # from src.db.storage.log_manager import LogManager
+    # idx  = IndexManager(base_path, store_name); idx.load()
+    # lmgr = LogManager(base_path, store_name)
+    # seg, offset, size = idx.get(record_id)
+    # raw = lmgr.read(seg, offset, size)
+    # result = _parse_flat_line(raw)
+    # cache.set(record_id, result)
+    # return result
+    raise NotImplementedError("Storage layer (WS-1) not yet implemented")
+
+
+def _multi_read(store_name: str, params: dict, cache: _WorkerCache, base_path: str) -> dict:
+    record_ids = params.get("record_ids", [])
+    results = {}
+    for rid in record_ids:
+        results[rid] = _read(store_name, {"record_id": rid}, cache, base_path)
+    return results
+
+
+def _exists(store_name: str, params: dict, cache: _WorkerCache, base_path: str) -> bool:
+    record_id = params.get("record_id", "")
+    if not record_id:
+        raise ValueError("record_id required for 'exists'")
+
+    if cache.get(record_id) is not None:
+        return True
+
+    # TODO(WS-1): check IndexManager._map
+    raise NotImplementedError("Storage layer (WS-1) not yet implemented")
+
+
+def _query(store_name: str, params: dict, cache: _WorkerCache, base_path: str) -> list:
+    # TODO(WS-3): route params["filter"] through QueryEngine
+    raise NotImplementedError("Query engine (WS-3) not yet implemented")
+
+
+def _count(store_name: str, params: dict, cache: _WorkerCache, base_path: str) -> int:
+    # TODO(WS-3): route params["filter"] through QueryEngine, return len
+    raise NotImplementedError("Query engine (WS-3) not yet implemented")
+
+
+# ---------------------------------------------------------------------------
+# Core worker loop — shared by process and thread entry points
+# ---------------------------------------------------------------------------
+
+def _worker_loop(
+    worker_id:          int,
+    store_name:         str,
+    base_path:          str,
+    work_queue,         # multiprocessing.Queue
+    results_queue,      # multiprocessing.Queue
+    invalidation_queue, # multiprocessing.Queue (private to this worker)
+    cache_max_size:     int,
+    cache_ttl:          float,
+    stop_event,         # multiprocessing.Event or threading.Event
+) -> None:
+    log = logging.getLogger(f"QueryWorker[{store_name}#{worker_id}]")
+    log.info("Worker started (pid=%s)", os.getpid())
+
+    cache = _WorkerCache(cache_max_size, cache_ttl)
+
+    while not stop_event.is_set():
+        # 1. Drain the private invalidation queue to keep cache consistent
+        #    with writes that the dispatcher has broadcast.
+        try:
+            while True:
+                record_id = invalidation_queue.get_nowait()
+                cache.delete(record_id)
+        except Exception:
+            pass  # queue.Empty or similar — expected
+
+        # 2. Pull the next task (short timeout so stop_event is checked)
+        try:
+            task_dict = work_queue.get(timeout=0.5)
+        except Exception:
+            continue  # timeout — loop back to check stop_event
+
+        # Sentinel signals graceful shutdown
+        if task_dict is WORKER_SENTINEL:
+            log.info("Received shutdown sentinel")
+            break
+
+        task_id = task_dict.get("task_id", "?")
+        log.debug("Handling task %s op=%s", task_id, task_dict.get("operation"))
+
+        # 3. Resolve and post result
+        try:
+            data = _resolve(task_dict, cache, base_path)
+            results_queue.put({
+                "task_id": task_id,
+                "status":  "ok",
+                "data":    data,
+            })
+        except Exception as exc:
+            log.warning("Task %s failed: %s", task_id, exc)
+            results_queue.put({
+                "task_id": task_id,
+                "status":  "error",
+                "error":   str(exc),
+            })
+
+    log.info("Worker exiting (pid=%s)", os.getpid())
+
+
+# ---------------------------------------------------------------------------
+# Entry points
+# ---------------------------------------------------------------------------
+
+def worker_process_main(
+    worker_id:          int,
+    project_root:       str,       # added to sys.path so imports resolve
+    store_name:         str,
+    base_path:          str,
+    work_queue,
+    results_queue,
+    invalidation_queue,
+    cache_max_size:     int,
+    cache_ttl:          float,
+    stop_event,
+) -> None:
+    """Entry point for multiprocessing.Process workers."""
+    # Ensure the project is importable inside the child process
+    if project_root not in sys.path:
+        sys.path.insert(0, project_root)
+
+    logging.basicConfig(level=logging.INFO)
+    _worker_loop(
+        worker_id, store_name, base_path,
+        work_queue, results_queue, invalidation_queue,
+        cache_max_size, cache_ttl, stop_event,
+    )
+
+
+def worker_thread_main(
+    worker_id:          int,
+    store_name:         str,
+    base_path:          str,
+    work_queue,
+    results_queue,
+    invalidation_queue,
+    cache_max_size:     int,
+    cache_ttl:          float,
+    stop_event,
+) -> None:
+    """Entry point for threading.Thread workers (parallel_enabled=False)."""
+    _worker_loop(
+        worker_id, store_name, base_path,
+        work_queue, results_queue, invalidation_queue,
+        cache_max_size, cache_ttl, stop_event,
+    )

src/db/storage/__init__.py

@@ -0,0 +1 @@
+

src/db/storage/blob_store.py

@@ -0,0 +1,47 @@
+"""
+BlobStore — stores oversized record values as individual binary/text files.
+
+Blob files live at: {store_path}/blobs/{record_id}.dat
+The pseudo-segment string returned is "blobs/{record_id}.dat", which
+LogManager.read() recognises and handles by reading the file directly.
+"""
+
+import os
+
+
+def is_blob(segment_str: str) -> bool:
+    """Return True if this segment string represents a blob file path."""
+    return segment_str.startswith("blobs/")
+
+
+def write_blob(store_path: str, record_id: str, data_str: str) -> tuple:
+    """
+    Write data_str to {store_path}/blobs/{record_id}.dat.
+
+    Returns (path_str, 0, byte_size) where path_str is the pseudo-segment
+    string "blobs/{record_id}.dat".
+    """
+    blobs_dir = os.path.join(store_path, "blobs")
+    os.makedirs(blobs_dir, exist_ok=True)
+    # Security: strip path separators from record_id
+    safe_id = record_id.replace("/", "_").replace("\\", "_").replace("..", "_")
+    file_path = os.path.join(blobs_dir, f"{safe_id}.dat")
+    encoded = data_str.encode("utf-8")
+    with open(file_path, "wb") as fh:
+        fh.write(encoded)
+    path_str = f"blobs/{safe_id}.dat"
+    return (path_str, 0, len(encoded))
+
+
+def read_blob(store_path: str, path_str: str) -> str:
+    """
+    Read and return the full content of a blob file.
+    path_str is the pseudo-segment string "blobs/{record_id}.dat".
+    """
+    # Security: validate path stays within store_path/blobs/
+    full_path = os.path.normpath(os.path.join(store_path, path_str))
+    blobs_dir = os.path.normpath(os.path.join(store_path, "blobs"))
+    if not full_path.startswith(blobs_dir + os.sep) and full_path != blobs_dir:
+        raise ValueError(f"Blob path traversal rejected: {path_str!r}")
+    with open(full_path, "rb") as fh:
+        return fh.read().decode("utf-8")

src/db/storage/index_manager.py

@@ -0,0 +1,92 @@
+"""
+IndexManager — manages the primary index for one store.
+
+Index file: {store_path}/{service}.idx
+Line format: {record_id}:{segment_seq}:{byte_offset}:{byte_size}
+Tombstone:   !{record_id}:{segment_seq}:{byte_offset}:{byte_size}
+
+Tombstoned entries are excluded from _map at load time; the physical .idx
+file is only fully rewritten during compaction via save_full().
+"""
+
+import os
+
+
+class IndexManager:
+    def __init__(self, store_path: str, service: str):
+        self.index_path = os.path.join(store_path, f"{service}.idx")
+        self._map: dict = {}   # record_id -> (segment_seq_str, offset, size)
+        self._dirty: bool = False
+        self.load()
+
+    # ------------------------------------------------------------------
+    # Load / save
+    # ------------------------------------------------------------------
+
+    def load(self) -> None:
+        """Parse the .idx file into _map, skipping tombstoned entries."""
+        if not os.path.exists(self.index_path):
+            return
+        with open(self.index_path, "r", encoding="utf-8") as fh:
+            for raw in fh:
+                line = raw.strip()
+                if not line:
+                    continue
+                if line.startswith("!"):
+                    # Tombstone — ensure it's absent from the map
+                    record_id = line[1:].split(":")[0]
+                    self._map.pop(record_id, None)
+                    continue
+                parts = line.split(":")
+                if len(parts) < 4:
+                    continue
+                # segment_seq may be a path like "blobs/abc.dat" containing "/"
+                # Format: record_id:segment_seq:offset:size
+                # We split on the last two colons to get offset and size safely
+                record_id = parts[0]
+                size = int(parts[-1])
+                offset = int(parts[-2])
+                segment_seq = ":".join(parts[1:-2])
+                self._map[record_id] = (segment_seq, offset, size)
+
+    def save_full(self) -> None:
+        """Rewrite the entire .idx file from _map (used after compaction)."""
+        tmp_path = self.index_path + ".tmp"
+        with open(tmp_path, "w", encoding="utf-8") as fh:
+            for record_id, (seg, offset, size) in self._map.items():
+                fh.write(f"{record_id}:{seg}:{offset}:{size}\n")
+        os.replace(tmp_path, self.index_path)
+        self._dirty = False
+
+    # ------------------------------------------------------------------
+    # CRUD
+    # ------------------------------------------------------------------
+
+    def get(self, record_id: str):
+        """Return (segment_seq, offset, size) or None."""
+        return self._map.get(record_id)
+
+    def set(self, record_id: str, segment_seq: str, offset: int, size: int) -> None:
+        """Update _map and append the new entry to the .idx file."""
+        self._map[record_id] = (segment_seq, offset, size)
+        with open(self.index_path, "a", encoding="utf-8") as fh:
+            fh.write(f"{record_id}:{segment_seq}:{offset}:{size}\n")
+        self._dirty = True
+
+    def delete(self, record_id: str) -> None:
+        """Remove from _map and write a tombstone to the .idx file."""
+        if record_id not in self._map:
+            return
+        entry = self._map.pop(record_id)
+        seg, offset, size = entry
+        with open(self.index_path, "a", encoding="utf-8") as fh:
+            fh.write(f"!{record_id}:{seg}:{offset}:{size}\n")
+        self._dirty = True
+
+    def records_in_segment(self, seq_str: str) -> list:
+        """Return all record IDs whose current segment matches seq_str."""
+        return [rid for rid, (seg, _, _) in self._map.items() if seg == seq_str]
+
+    def all_record_ids(self) -> list:
+        """Return all live record IDs."""
+        return list(self._map.keys())

src/db/storage/log_manager.py

@@ -0,0 +1,119 @@
+"""
+LogManager — append-only rolling log file manager for one store.
+
+Segment files: {store_path}/{service}_{NNN}.log   (NNN = zero-padded 3-digit int)
+On init, scans store_path for existing segments and opens the highest one
+in append mode.  When a segment exceeds segment_threshold bytes it is closed
+and a new one is started (_rollover).
+"""
+
+import os
+import re
+
+
+class LogManager:
+    def __init__(self, store_path: str, service: str, segment_threshold: int):
+        """
+        Parameters
+        ----------
+        store_path         : absolute path to the store directory
+        service            : store name (used as the filename prefix)
+        segment_threshold  : byte size at which a new segment is opened
+        """
+        self.store_path = store_path
+        self.service = service
+        self.segment_threshold = segment_threshold
+
+        self.active_segment: int = self._find_highest_segment()
+        seg_path = self.segment_path(self.active_segment)
+        self.active_fh = open(seg_path, "ab")
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _find_highest_segment(self) -> int:
+        """Return the highest existing segment number, or 1 if none exist."""
+        pattern = re.compile(rf"^{re.escape(self.service)}_(\d{{3}})\.log$")
+        highest = 0
+        for name in os.listdir(self.store_path):
+            m = pattern.match(name)
+            if m:
+                n = int(m.group(1))
+                if n > highest:
+                    highest = n
+        if highest == 0:
+            highest = 1
+            open(self.segment_path(1), "ab").close()
+        return highest
+
+    def segment_path(self, seq_int: int) -> str:
+        """Return the absolute path for segment number seq_int."""
+        return os.path.join(self.store_path, f"{self.service}_{seq_int:03d}.log")
+
+    def _rollover(self) -> None:
+        """Close the current segment and open a new one."""
+        self.active_fh.flush()
+        self.active_fh.close()
+        self.active_segment += 1
+        self.active_fh = open(self.segment_path(self.active_segment), "ab")
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def append(self, record_id: str, flat_line_str: str) -> tuple:
+        """
+        Write one record line to the active segment.
+
+        Parameters
+        ----------
+        record_id     : the record's ID (used only for return value metadata)
+        flat_line_str : already-serialized line string (no trailing newline)
+
+        Returns
+        -------
+        (segment_seq_str, byte_offset, byte_size)
+        """
+        encoded = (flat_line_str + "\n").encode("utf-8")
+        offset = self.active_fh.tell()
+        self.active_fh.write(encoded)
+        self.active_fh.flush()
+        size = len(encoded)
+        seg_str = f"{self.active_segment:03d}"
+        if self.active_fh.tell() >= self.segment_threshold:
+            self._rollover()
+        return (seg_str, offset, size)
+
+    def read(self, segment_seq_str: str, offset: int, size: int) -> str:
+        """
+        Read exactly `size` bytes from the indicated segment at `offset`.
+        Uses stored size — does NOT use readline to avoid partial reads.
+        Returns the decoded string with trailing newline stripped.
+        """
+        # Handle blob entries (segment_seq_str starts with "blobs/")
+        if segment_seq_str.startswith("blobs/"):
+            blob_path = os.path.join(self.store_path, segment_seq_str)
+            with open(blob_path, "r", encoding="utf-8") as fh:
+                return fh.read()
+        seq_int = int(segment_seq_str)
+        path = self.segment_path(seq_int)
+        with open(path, "rb") as fh:
+            fh.seek(offset)
+            return fh.read(size).decode("utf-8").rstrip("\n")
+
+    def list_segments(self) -> list:
+        """Return all segment numbers found on disk, sorted ascending."""
+        pattern = re.compile(rf"^{re.escape(self.service)}_(\d{{3}})\.log$")
+        nums = []
+        for name in os.listdir(self.store_path):
+            m = pattern.match(name)
+            if m:
+                nums.append(int(m.group(1)))
+        return sorted(nums)
+
+    def close(self) -> None:
+        """Flush and close the active file handle."""
+        if self.active_fh and not self.active_fh.closed:
+            self.active_fh.flush()
+            self.active_fh.close()

src/db/storage/serializer.py

@@ -0,0 +1,38 @@
+"""
+Record serialization helpers for .log file lines.
+
+Wire format:  {record_id} {json_object}
+One line per record: record_id (no spaces) followed by a single space,
+then a compact JSON object.  Native JSON types (str, int, float, bool, list,
+dict, None) are preserved exactly — no custom encoding required.
+"""
+
+import json
+
+
+def serialize_record(record_id: str, flat_dict: dict) -> str:
+    """
+    Produce a single log line string (no trailing newline).
+    Example: abc123 {"name": "Gizmo", "price": 19.99, "in_stock": true}
+    """
+    return f"{record_id} {json.dumps(flat_dict, ensure_ascii=False)}"
+
+
+def deserialize_record(line: str) -> tuple:
+    """
+    Parse a log line into (record_id, flat_dict).
+    Strips trailing newline/whitespace before parsing.
+    Returns (None, {}) on empty or malformed lines.
+    """
+    line = line.strip()
+    if not line:
+        return (None, {})
+    space = line.find(" ")
+    if space == -1:
+        return (None, {})
+    record_id = line[:space]
+    try:
+        flat_dict = json.loads(line[space + 1:])
+    except json.JSONDecodeError:
+        return (None, {})
+    return (record_id, flat_dict)

src/filing/__init__.py

@@ -0,0 +1,31 @@
+import json
+
+def read_file(path):
+    with open(path, 'r', encoding='utf-8') as f:
+        return f.read()
+    
+def write_file(path, content):
+    with open(path, 'w', encoding='utf-8') as f:
+        f.write(content)
+
+def append_file(path, content):
+    with open(path, 'a', encoding='utf-8') as f:
+        f.write(content)
+
+def delete_file(path):
+    import os
+    os.remove(path)
+
+def file_exists(path):
+    import os
+    return os.path.exists(path)
+
+def list_files(directory):
+    import os
+    return os.listdir(directory)
+
+def read_json(path):
+    return json.loads(read_file(path))
+
+def write_json(path, data):
+    write_file(path, json.dumps(data, indent=4))