PyMkDB 0.1.0__py3-none-any.whl

sdk/mkdb_client.py

@@ -0,0 +1,226 @@
+"""
+MkDB high-level SDK client.
+
+Usage:
+    from sdk.mkdb_client import MkDBClient
+
+    client = MkDBClient(host="127.0.0.1", port=9001)
+    client.connect()
+
+    client.set("products_1v", "p001", {"name": "Widget", "price": 9.99})
+    record = client.get("products_1v", "p001")
+    print(record)
+
+    results = client.query("products_1v", {"price": {"gte": 5, "lte": 20}})
+    print(results)
+
+    client.on_update("products_1v", lambda event: print("Change:", event))
+    client.close()
+"""
+
+import hmac
+import hashlib
+import secrets
+from typing import Callable, Optional
+
+from sdk.connection import Connection
+from sdk.http_connection import HttpConnection
+from sdk.delta import flatten
+from sdk.responses import (
+    GetResponse,
+    WriteResponse,
+    DeleteResponse,
+    QueryResponse,
+    GenerateIdResponse,
+)
+
+
+class MkDBClient:
+    def __init__(self, host: str = "127.0.0.1", port: int = 9001,
+                 access: str = "R",
+                 username: str = "",
+                 password: str = "mk_db",
+                 recv_timeout: float = 30.0,
+                 transport: str = "socket"):
+        """
+        Parameters
+        ----------
+        host, port      : server address
+        access          : "R", "W", or "RW"
+        username        : username (required if server has users configured)
+        password        : password / write key
+        recv_timeout    : seconds to wait for a response
+        transport       : "socket" (default) or "http"
+                          HTTP transport does not support pub-sub (on_update).
+        """
+        t = transport.lower()
+        if t == "http":
+            self._conn = HttpConnection(host=host, port=port, recv_timeout=recv_timeout,
+                                        access=access, username=username, password=password)
+        else:
+            self._conn = Connection(host=host, port=port, recv_timeout=recv_timeout,
+                                    access=access, username=username, password=password)
+        self.auth = AuthManager(self)
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    def connect(self) -> None:
+        self._conn.connect()
+
+    def close(self) -> None:
+        self._conn.close()
+
+    # ------------------------------------------------------------------
+    # Data operations
+    # ------------------------------------------------------------------
+
+    def get(self, store: str, record_id: str) -> GetResponse:
+        """Read a record. Returns a GetResponse (check .found before using .data)."""
+        resp = self._conn.send({
+            "action": "read",
+            "store":  store,
+            "record_id": record_id,
+        })
+        self._raise_if_error(resp)
+        data = resp.get("data")
+        return GetResponse(record_id=record_id, data=data, found=data is not None)
+
+    def set(self, store: str, record_id: str, delta: dict,
+            flatten_nested: bool = True) -> WriteResponse:
+        """Write / update a record. Returns a WriteResponse."""
+        flat = flatten(delta) if flatten_nested else delta
+        resp = self._conn.send({
+            "action":    "write",
+            "store":     store,
+            "record_id": record_id,
+            "delta":     flat,
+        })
+        self._raise_if_error(resp)
+        rid = (resp.get("data") or {}).get("record_id", record_id)
+        return WriteResponse(record_id=rid, store=store)
+
+    def insert(self, store: str, delta: dict,
+               flatten_nested: bool = True) -> WriteResponse:
+        """Write a new record with a server-generated ID. Returns a WriteResponse."""
+        flat = flatten(delta) if flatten_nested else delta
+        resp = self._conn.send({
+            "action": "write",
+            "store":  store,
+            "delta":  flat,
+        })
+        self._raise_if_error(resp)
+        rid = (resp.get("data") or {}).get("record_id", "")
+        return WriteResponse(record_id=rid, store=store)
+
+    def generate_id(self, store: str) -> GenerateIdResponse:
+        """Ask the server to generate a unique ID for the store without writing anything."""
+        resp = self._conn.send({
+            "action": "generate_id",
+            "store":  store,
+        })
+        self._raise_if_error(resp)
+        rid = (resp.get("data") or {}).get("record_id", "")
+        return GenerateIdResponse(record_id=rid, store=store)
+
+    def delete(self, store: str, record_id: str) -> DeleteResponse:
+        """Delete a record. Returns a DeleteResponse."""
+        resp = self._conn.send({
+            "action":    "delete",
+            "store":     store,
+            "record_id": record_id,
+        })
+        self._raise_if_error(resp)
+        return DeleteResponse(record_id=record_id, store=store)
+
+    def query(self, store: str, filter_dict: dict,
+              hydrate: bool = False) -> QueryResponse:
+        """Query a store. Returns a QueryResponse."""
+        resp = self._conn.send({
+            "action":  "query",
+            "store":   store,
+            "filter":  filter_dict,
+            "hydrate": hydrate,
+        })
+        self._raise_if_error(resp)
+        data = resp.get("data") or {}
+        return QueryResponse(
+            count=data.get("count", 0),
+            ids=data.get("ids", []),
+            records=data.get("records"),
+            store=store,
+        )
+
+    # ------------------------------------------------------------------
+    # Pub-sub
+    # ------------------------------------------------------------------
+
+    def on_update(self, store: str, callback: Callable[[dict], None]) -> None:
+        """
+        Subscribe to updates for a store and register a callback for push events.
+        The callback receives the full server-push event dict.
+        """
+        def _filter(event: dict) -> None:
+            if event.get("store") == store:
+                callback(event)
+        self._conn.register_push_handler(_filter)
+        self._conn.send_raw({"type": "subscribe", "store": store})
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _raise_if_error(resp: dict) -> None:
+        if resp.get("status") != "ok":
+            raise RuntimeError(resp.get("error", "Unknown error"))
+
+
+class AuthManager:
+    """
+    Simple credential store using MkDB itself as the backing store.
+
+    Passwords are never stored in plaintext.
+    Storage format (flat dict in MkDB):
+      {"auth.salt": "<hex>", "auth.hash": "<sha256 hex>"}
+    Stored under store="__auth__", record_id=username.
+    """
+
+    AUTH_STORE = "__auth__"
+
+    def __init__(self, client: MkDBClient):
+        self._client = client
+
+    def register(self, username: str, password: str) -> None:
+        """
+        Register a new user. Raises RuntimeError if username already exists
+        or if the __auth__ store is unavailable.
+        """
+        salt = secrets.token_hex(32)
+        pw_hash = hashlib.sha256(
+            (password + salt).encode("utf-8")
+        ).hexdigest()
+        self._client.set(
+            self.AUTH_STORE,
+            username,
+            {"auth.salt": salt, "auth.hash": pw_hash},
+            flatten_nested=False,
+        )
+
+    def validate(self, username: str, password: str) -> bool:
+        """
+        Validate credentials. Returns True if valid, False otherwise.
+        Uses hmac.compare_digest to prevent timing attacks.
+        """
+        record = self._client.get(self.AUTH_STORE, username)
+        if record is None:
+            # Use compare_digest anyway to avoid timing oracle
+            hmac.compare_digest("x", "y")
+            return False
+        stored_salt = record.get("auth.salt", "")
+        stored_hash = record.get("auth.hash", "")
+        computed = hashlib.sha256(
+            (password + stored_salt).encode("utf-8")
+        ).hexdigest()
+        return hmac.compare_digest(computed, stored_hash)

sdk/responses.py

@@ -0,0 +1,154 @@
+"""
+MkDB SDK response objects.
+
+Every public client method returns one of these instead of a raw dict,
+giving callers full IDE autocompletion and type-checking.
+
+Entity data (the actual record fields) is intentionally left as plain
+``dict`` — no generated model classes.
+"""
+
+from __future__ import annotations
+from typing import Optional, Union
+
+
+# ---------------------------------------------------------------------------
+# Minimal base_object — copied from src/objects so the SDK stays standalone.
+# Provides .json and .update() exactly as the server-side version does.
+# ---------------------------------------------------------------------------
+
+def _format_items(input: Union[dict, list]) -> Union[dict, list]:
+    if type(input) == list:
+        data = []
+        for item in input:
+            if isinstance(item, base_object):
+                data.append(item.json)
+            elif type(item) in [list, dict]:
+                data.append(_format_items(item))
+            elif type(item) in [tuple, int, float, str, bool]:
+                data.append(item)
+        return data
+    elif type(input) == dict:
+        data = {}
+        for key in input:
+            if "__" in str(key):
+                continue
+            item = input[key]
+            if isinstance(item, base_object):
+                data[key] = item.json
+            elif type(item) in [list, dict]:
+                data[key] = _format_items(item)
+            elif type(item) in [tuple, int, float, str, bool]:
+                data[key] = item
+        return data
+    return input
+
+
+class base_object:
+    def __init__(self, data: dict = {}):
+        self.update(data)
+
+    def get(self, name, default=None):
+        return self.__dict__.get(name, default)
+
+    @property
+    def json(self) -> dict:
+        data = {}
+        for name in self.__dict__:
+            if "__" in name:
+                continue
+            item = self.__dict__[name]
+            if isinstance(item, base_object):
+                data[name] = item.json
+            elif type(item) in [list, dict]:
+                data[name] = _format_items(item)
+            elif type(item) in [tuple, int, float, str, bool]:
+                data[name] = item
+        return data
+
+    def update(self, data: dict):
+        if not isinstance(data, dict):
+            return
+        for name, item in data.items():
+            if name in self.__dict__:
+                if isinstance(self.__dict__[name], base_object):
+                    self.__dict__[name].update(item)
+                elif type(item) in [list, int, float, str, dict, bool]:
+                    setattr(self, name, item)
+
+    def __repr__(self) -> str:
+        return str(self.json)
+
+
+# ---------------------------------------------------------------------------
+# Response classes
+# ---------------------------------------------------------------------------
+
+
+class GetResponse(base_object):
+    """Returned by ``client.get()``."""
+
+    def __init__(self, record_id: str, data: Optional[dict], found: bool):
+        self.record_id: str            = record_id
+        self.data:      Optional[dict] = data   # flat field dict, or None
+        self.found:     bool           = found
+        super().__init__({})
+
+    def __bool__(self) -> bool:
+        return self.found
+
+
+class WriteResponse(base_object):
+    """Returned by ``client.set()`` and ``client.insert()``."""
+
+    def __init__(self, record_id: str, store: str):
+        self.record_id: str = record_id
+        self.store:     str = store
+        super().__init__({})
+
+
+class DeleteResponse(base_object):
+    """Returned by ``client.delete()``."""
+
+    def __init__(self, record_id: str, store: str):
+        self.record_id: str = record_id
+        self.store:     str = store
+        super().__init__({})
+
+
+class QueryResponse(base_object):
+    """Returned by ``client.query()``."""
+
+    def __init__(self, count: int, ids: list,
+                 records: Optional[list] = None,
+                 store: str = ""):
+        self.count:   int            = count
+        self.ids:     list           = ids
+        # Only populated when hydrate=True
+        self.records: Optional[list] = records
+        self.store:   str            = store
+        super().__init__({})
+
+    def __len__(self) -> int:
+        return self.count
+
+    def __bool__(self) -> bool:
+        return self.count > 0
+
+    def __iter__(self):
+        """Iterate over records if hydrated, otherwise over IDs."""
+        if self.records is not None:
+            return iter(self.records)
+        return iter(self.ids)
+
+
+class GenerateIdResponse(base_object):
+    """Returned by ``client.generate_id()``."""
+
+    def __init__(self, record_id: str, store: str):
+        self.record_id: str = record_id
+        self.store:     str = store
+        super().__init__({})
+
+    def __str__(self) -> str:
+        return self.record_id

src/__init__.py

@@ -0,0 +1 @@
+# MkDB core engine package

src/config/db.py

@@ -0,0 +1,227 @@
+import os
+
+from src.filing import write_json
+from src.config.server import control_server, socket_server, http_server
+from src.objects import base_object
+
+class query_worker_config(base_object):
+    def __init__(self, data:dict={}):
+        self.parallel_enabled = False   # set True to spin up worker processes
+        self.worker_count = 0           # 0 = auto (os.cpu_count())
+        self.worker_cache_size = 500    # max records in each worker's local cache
+        self.worker_cache_ttl = 30      # seconds before a worker cache entry expires
+        self.task_timeout = 30.0        # seconds before submit() raises TimeoutError
+        super().__init__(data)
+
+class ram_config(base_object):
+    def __init__(self, data:dict={}):
+        self.max_size = 1000
+        self.ttl = 3600
+        self.cleanup_interval = 600
+        self.clean_type = "lru"
+        self.update_on_access = True
+        self.index = True
+        self.index_cache_ttl = 3600
+        self.query_fields = True
+        self.query_fields_cache_ttl = 3600
+        # 0 = keep all query indexes in RAM.
+        # >0 = if an index file on disk exceeds this many bytes, don't load
+        #      it into RAM; query and write directly from/to the file instead.
+        self.index_ram_threshold_bytes = 0
+        super().__init__(data)
+
+TOKEN_SOFT_LIMIT = 28   # health = "low"  when auto_expand is on and length >= this
+TOKEN_HARD_LIMIT = 32   # health = "degraded" regardless; performance warning threshold
+
+class entity_config(base_object):
+    def __init__(self, data:dict={}):
+        self.token_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+        self.token_length = 16
+        self.auto_expand = False
+        super().__init__(data)
+
+    @property
+    def health(self) -> dict:
+        """Return entity-config health status and an explanatory message."""
+        if self.token_length > TOKEN_HARD_LIMIT:
+            return {
+                "status": "degraded",
+                "message": (
+                    f"Token length {self.token_length} exceeds the recommended "
+                    f"maximum of {TOKEN_HARD_LIMIT} characters. "
+                    "Performance may degrade — consider reducing it."
+                ),
+            }
+        if self.auto_expand and self.token_length >= TOKEN_SOFT_LIMIT:
+            return {
+                "status": "low",
+                "message": (
+                    f"Token length {self.token_length} is approaching the "
+                    f"{TOKEN_HARD_LIMIT}-character performance limit. "
+                    "With auto expand enabled this may increase further, "
+                    "degrading database performance."
+                ),
+            }
+        return {"status": "ok", "message": ""}
+
+class file_config(base_object):
+    def __init__(self, data:dict={}):
+        self.file_size_split_trigger = 20000000
+        self.type = "rolling_log"
+        self.blob_threshold        = 5 * 1024 * 1024   # 5 MB
+        self.segment_threshold     = 50 * 1024 * 1024  # 50 MB
+        self.parity_nsym           = 10
+        self.compaction_dead_ratio = 0.3
+        self.compaction_interval   = 60
+        super().__init__(data)
+
+class field_schema(base_object):
+    def __init__(self, data={}):
+        self.name        = ""
+        self.queryable   = False    # False | "exact" | "full-text" | "numeric"
+        self.indexed     = False
+        super().__init__(data)
+
+class schema_config(base_object):
+    def __init__(self, data={}):
+        self.fields: dict = {}
+        super().__init__(data)
+        self._normalise_fields()
+
+    def _normalise_fields(self):
+        self.fields = {
+            k: field_schema(v) if isinstance(v, dict) else v
+            for k, v in self.fields.items()
+        }
+
+    def update(self, data: dict):
+        super().update(data)
+        self._normalise_fields()
+
+class write_queue_config(base_object):
+    def __init__(self, data={}):
+        self.debounce_window = 5.0
+        self.max_pending     = 10000
+        super().__init__(data)
+
+class store_rate_limit(base_object):
+    """Per-store IP rate limit for the HTTP data plane."""
+    def __init__(self, data: dict = {}):
+        self.enabled = False
+        self.max_requests_per_second = 100
+        super().__init__(data)
+
+class store_config(base_object):
+    def __init__(self, data:dict={}):
+        self.name = ""
+        self.description = ""
+        self.protect_reads = False          # when True, read/query also require auth
+        self.slow_query_threshold_ms = 0.0  # 0 = disabled; >0 = log queries slower than X ms
+        self.client_id_header = ""          # HTTP header to use as client identifier; "" = remote_addr
+        self.file_config = file_config(data.get("file_config", {}))
+        self.entity_config = entity_config(data.get("entity_config", {}))
+        self.ram_config = ram_config(data.get("ram_config", {}))
+        self.query_worker_config = query_worker_config(data.get("query_worker_config", {}))
+        self.schema_config      = schema_config(data.get("schema_config", {}))
+        self.write_queue_config = write_queue_config(data.get("write_queue_config", {}))
+        self.rate_limit         = store_rate_limit(data.get("rate_limit", {}))
+        super().__init__(data)
+
+class backup_config(base_object):
+    def __init__(self, data:dict={}):
+        self.enabled = False
+        self.interval = 3600
+        self.directory = "backups"
+        super().__init__(data)
+
+class store_permission(base_object):
+    """Read/write permissions for a user on a specific store."""
+    def __init__(self, data: dict = {}):
+        self.read  = True
+        self.write = False
+        super().__init__(data)
+
+class db_user(base_object):
+    """A database user who can authenticate against the data plane."""
+    def __init__(self, data: dict = {}):
+        self.username      = ""
+        self.password_hash = ""
+        self.stores: dict  = {}   # store_name -> store_permission
+        super().__init__(data)
+        self.stores = {
+            k: store_permission(v) if isinstance(v, dict) else v
+            for k, v in self.stores.items()
+        }
+
+class data_security_config(base_object):
+    """Security settings for the data plane (HTTP + Socket)."""
+    def __init__(self, data: dict = {}):
+        # When True, GET/read/query requests also require authentication.
+        # When False (default), only write/delete requests require auth.
+        self.protect_reads = False
+        super().__init__(data)
+
+# ── Control-plane RBAC ───────────────────────────────────────────────────────
+# Roles (least → most privileged):
+#   viewer   – read-only: list stores, get configs, view logs
+#   operator – viewer + create/delete stores, update store configs
+#   admin    – operator + all control settings (auth, users, server config)
+
+CONTROL_ROLES = ("viewer", "operator", "admin")
+
+class control_user(base_object):
+    """A control-plane SDK user with a role."""
+    def __init__(self, data: dict = {}):
+        self.username      = ""
+        self.password_hash = ""
+        self.role          = "viewer"   # "viewer" | "operator" | "admin"
+        super().__init__(data)
+
+class mkdb_servers_config(base_object):
+    def __init__(self, data:dict={}):
+        self.control_server = control_server(data.get("control_server", {}))
+        self.socket_server = socket_server(data.get("socket_server", {}))
+        self.http_server = http_server(data.get("http_server", {}))
+        super().__init__(data)
+
+class mkdb_config(base_object):
+    def __init__(self, data:dict={}):
+        self.name = ""
+        self.storage_nodes: list = []
+        self.stores:dict[str, store_config] = {}
+        super().__init__(data)
+        self.servers = mkdb_servers_config(data.get("servers", {}))
+        self.backup = backup_config(data.get("backup", {}))
+        for store_name, store_data in data.get("stores", {}).items():
+            self.stores[store_name] = store_config(store_data)
+        self.users: dict = {}   # username -> db_user
+        for uname, udata in data.get("users", {}).items():
+            self.users[uname] = db_user(udata)
+        self.data_security = data_security_config(data.get("data_security", {}))
+        self.control_users: dict = {}   # username -> control_user
+        for uname, udata in data.get("control_users", {}).items():
+            self.control_users[uname] = control_user(udata)
+
+    @property
+    def base_path(self):
+        if not hasattr(self, "__base_path__"):
+            self.__base_path__ = os.getcwd()
+        return self.__base_path__
+
+    def new_store(self, store_name:str, description:str=""):
+        """Add a new store to the configuration."""
+        if store_name in self.stores:
+            raise ValueError(f"Store '{store_name}' already exists in the configuration.")
+        
+        new_store_config = store_config()
+        new_store_config.name = store_name
+        new_store_config.description = description
+        self.stores[store_name] = new_store_config
+        print(f"Store '{store_name}' added to configuration.")
+        self.save()
+
+    def save(self):
+        """Save the current configuration to a file."""
+        config_path = os.path.join(self.base_path, "config.json")
+        write_json(config_path, self.json)
+        print(f"Configuration saved to {config_path}")

src/config/server.py

@@ -0,0 +1,52 @@
+from src.objects import base_object
+
+class server_address_config(base_object):
+    def __init__(self, data:dict={}):
+        self.host = ""
+        self.port = 0
+        super().__init__(data)
+
+
+class auth_config(base_object):
+    """Authentication configuration shared by all server types."""
+    def __init__(self, data:dict={}):
+        self.enabled = False
+        self.password_hash = ""   # empty = no password set; format: pbkdf2:sha256:{iters}:{salt}:{hash}
+        self.session_ttl = 3600   # seconds until a session token expires
+        super().__init__(data)
+
+
+class control_server(base_object):
+    def __init__(self, data:dict={}):
+        self.enabled = True
+        self.address = server_address_config(data.get("address", {}))
+        self.auth = auth_config(data.get("auth", {}))
+        super().__init__(data)
+        if self.enabled and self.address.host in [None, ""]:
+            print("Control server enabled but no host specified. Defaulting to localhost.")
+            self.address.host = "localhost"
+        if self.enabled and self.address.port == 0:
+            print("Control server enabled but no port specified. Defaulting to port 80.")
+            self.address.port = 80
+
+
+class socket_server(base_object):
+    def __init__(self, data:dict={}):
+        self.enabled = True
+        self.address = server_address_config(data.get("address", {}))
+        self.auth = auth_config(data.get("auth", {}))
+        self.heartbeat_interval = 5.0
+        self.max_clients = 100
+        self.recv_timeout = 30.0
+        super().__init__(data)
+
+class http_server(base_object):
+    def __init__(self, data:dict={}):
+        self.enabled = False
+        self.address = server_address_config(data.get("address", {}))
+        self.auth = auth_config(data.get("auth", {}))
+        self.max_body_size = 10 * 1024 * 1024
+        self.cors_enabled = False
+        self.cors_origins: list = ["*"]
+        self.max_requests_per_second = 100
+        super().__init__(data)