PyMkDB 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. pymkdb-0.1.0/PKG-INFO +86 -0
  2. pymkdb-0.1.0/PyMkDB.egg-info/PKG-INFO +86 -0
  3. pymkdb-0.1.0/PyMkDB.egg-info/SOURCES.txt +57 -0
  4. pymkdb-0.1.0/PyMkDB.egg-info/dependency_links.txt +1 -0
  5. pymkdb-0.1.0/PyMkDB.egg-info/entry_points.txt +2 -0
  6. pymkdb-0.1.0/PyMkDB.egg-info/requires.txt +2 -0
  7. pymkdb-0.1.0/PyMkDB.egg-info/top_level.txt +3 -0
  8. pymkdb-0.1.0/README.md +63 -0
  9. pymkdb-0.1.0/pymkdb/__init__.py +6 -0
  10. pymkdb-0.1.0/pymkdb/cli.py +57 -0
  11. pymkdb-0.1.0/pyproject.toml +40 -0
  12. pymkdb-0.1.0/sdk/__init__.py +1 -0
  13. pymkdb-0.1.0/sdk/connection.py +225 -0
  14. pymkdb-0.1.0/sdk/delta.py +19 -0
  15. pymkdb-0.1.0/sdk/http_connection.py +180 -0
  16. pymkdb-0.1.0/sdk/mkdb_client.py +226 -0
  17. pymkdb-0.1.0/sdk/responses.py +154 -0
  18. pymkdb-0.1.0/setup.cfg +4 -0
  19. pymkdb-0.1.0/src/__init__.py +1 -0
  20. pymkdb-0.1.0/src/config/db.py +227 -0
  21. pymkdb-0.1.0/src/config/server.py +52 -0
  22. pymkdb-0.1.0/src/db/__init__.py +207 -0
  23. pymkdb-0.1.0/src/db/cache/__init__.py +1 -0
  24. pymkdb-0.1.0/src/db/cache/ram_cache.py +144 -0
  25. pymkdb-0.1.0/src/db/cache/write_queue.py +156 -0
  26. pymkdb-0.1.0/src/db/maintenance/__init__.py +0 -0
  27. pymkdb-0.1.0/src/db/maintenance/compactor.py +118 -0
  28. pymkdb-0.1.0/src/db/maintenance/task_scheduler.py +73 -0
  29. pymkdb-0.1.0/src/db/objects/store.py +283 -0
  30. pymkdb-0.1.0/src/db/parity/__init__.py +0 -0
  31. pymkdb-0.1.0/src/db/parity/parity_manager.py +196 -0
  32. pymkdb-0.1.0/src/db/query/__init__.py +1 -0
  33. pymkdb-0.1.0/src/db/query/full_text_index.py +168 -0
  34. pymkdb-0.1.0/src/db/query/numeric_index.py +196 -0
  35. pymkdb-0.1.0/src/db/query/query_engine.py +308 -0
  36. pymkdb-0.1.0/src/db/query/tokenizer.py +48 -0
  37. pymkdb-0.1.0/src/db/query_workers/__init__.py +16 -0
  38. pymkdb-0.1.0/src/db/query_workers/dispatcher.py +339 -0
  39. pymkdb-0.1.0/src/db/query_workers/task.py +78 -0
  40. pymkdb-0.1.0/src/db/query_workers/worker.py +292 -0
  41. pymkdb-0.1.0/src/db/requesting/main.py +0 -0
  42. pymkdb-0.1.0/src/db/storage/__init__.py +1 -0
  43. pymkdb-0.1.0/src/db/storage/blob_store.py +47 -0
  44. pymkdb-0.1.0/src/db/storage/index_manager.py +92 -0
  45. pymkdb-0.1.0/src/db/storage/log_manager.py +119 -0
  46. pymkdb-0.1.0/src/db/storage/serializer.py +38 -0
  47. pymkdb-0.1.0/src/filing/__init__.py +31 -0
  48. pymkdb-0.1.0/src/objects/__init__.py +190 -0
  49. pymkdb-0.1.0/src/runtime/__init__.py +15 -0
  50. pymkdb-0.1.0/src/server/__init__.py +0 -0
  51. pymkdb-0.1.0/src/server/coms/actions.py +209 -0
  52. pymkdb-0.1.0/src/server/coms/http.py +46 -0
  53. pymkdb-0.1.0/src/server/coms/http_handlers.py +445 -0
  54. pymkdb-0.1.0/src/server/coms/metrics.py +231 -0
  55. pymkdb-0.1.0/src/server/coms/socket.py +461 -0
  56. pymkdb-0.1.0/src/server/coms/socket_protocol.py +54 -0
  57. pymkdb-0.1.0/src/server/control/api/actions.py +1001 -0
  58. pymkdb-0.1.0/src/server/control/server.py +404 -0
  59. pymkdb-0.1.0/src/server/event_log.py +58 -0
pymkdb-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,86 @@
1
+ Metadata-Version: 2.4
2
+ Name: PyMkDB
3
+ Version: 0.1.0
4
+ Summary: A log-structured, partitioned NoSQL database engine with full-text search, numeric indexes, and dual TCP/HTTP protocols.
5
+ Author: MNG
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/MNG/MkDB
8
+ Project-URL: Repository, https://github.com/MNG/MkDB
9
+ Keywords: database,nosql,document-store,full-text-search,log-structured
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.10
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Topic :: Database
18
+ Classifier: Topic :: Database :: Database Engines/Servers
19
+ Requires-Python: >=3.10
20
+ Description-Content-Type: text/markdown
21
+ Requires-Dist: colorama>=0.4.6
22
+ Requires-Dist: reedsolo>=1.7.0
23
+
24
+ # MkDB
25
+
26
+ MkDB is a Custom Log-Structured Merge & Partitioned Redundant Storage Engine built entirely in Python. It provides a robust, highly-available NoSQL/Document database experience with secondary indexing, full-text search, and dual-protocol network access.
27
+
28
+ ## Architecture Highlights
29
+
30
+ - **Rolling Log Storage Engine**: Append-only storage format guaranteeing high write availability with safe background compaction.
31
+ - **RAM Cache & Debounced Write Queue**: In-memory caching and debounced batching for extreme performance under high write load.
32
+ - **Query Engine & Secondary Indexes**: Fully featured query evaluation including numeric range checks and tokenized full-text inverted indexes.
33
+ - **Data Integrity**: Multi-disk mirroring and Reed-Solomon parity encoding for proactive self-healing and failover.
34
+ - **Dual Protocols**: Accessible via high-speed, persistent TCP WebSockets or standard stateless REST HTTP endpoints.
35
+ - **Web Administration UI**: Includes an embedded web control panel out-of-the-box (`/control` endpoint).
36
+
37
+ ## Project Structure
38
+
39
+ - `src/db/`: The core database engine (storage primitives, RAM caching, query evaluator, auto-compaction and parity management).
40
+ - `src/server/`: The networking boundary. Houses the TCP Socket and HTTP REST servers, as well as the web-based Control Panel.
41
+ - `src/config/`: Configuration schemas for tailoring memory limits, storage thresholds, and cluster layout.
42
+ - `sdk/`: The official `MkDBClient` for programmatic interaction from Python code.
43
+
44
+ ## Getting Started
45
+
46
+ ### Prerequisites
47
+ - Python 3.10+
48
+ - The database storage format is built into MkDB natively, but you'll need the following for advanced data integrity features (Reed-Solomon logic):
49
+ ```bash
50
+ pip install reedsolo
51
+ ```
52
+
53
+ ### Starting the Server
54
+ MkDB operates as a CLI tool. Launch the engine by pointing it to your desired database directory (which must contain a `config.json` file configuring your stores and network bindings):
55
+ ```bash
56
+ mkdb /path/to/your/db
57
+ ```
58
+ Once running, the database will host both TCP socket and HTTP interfaces as specified in your `config.json`. The web control panel is accessible via your browser (check server output for the bound port, normally `http://localhost:<port>`).
59
+
60
+ ### Using the Python SDK
61
+ The `MkDBClient` connects seamlessly to your database and abstracts the dual-protocol system:
62
+
63
+ ```python
64
+ from sdk.mkdb_client import MkDBClient
65
+
66
+ client = MkDBClient()
67
+ client.connect(host="127.0.0.1", port=8080)
68
+
69
+ # Writing a document (computes delta updates intelligently)
70
+ client.set(
71
+ store="products",
72
+ record_id="prod_001",
73
+ data={"name": "Steel Bolt", "price": 9.99, "category": "fasteners"}
74
+ )
75
+
76
+ # Reading a document
77
+ record = client.get("products", "prod_001")
78
+
79
+ # Querying with filters
80
+ results = client.query("products", filter={
81
+ "price": {"<=": 10.00},
82
+ "category": ["fasteners"]
83
+ })
84
+ ```
85
+
86
+ See the `docs/` folder for comprehensive guides on the Query Syntax and SDK Reference.
@@ -0,0 +1,86 @@
1
+ Metadata-Version: 2.4
2
+ Name: PyMkDB
3
+ Version: 0.1.0
4
+ Summary: A log-structured, partitioned NoSQL database engine with full-text search, numeric indexes, and dual TCP/HTTP protocols.
5
+ Author: MNG
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/MNG/MkDB
8
+ Project-URL: Repository, https://github.com/MNG/MkDB
9
+ Keywords: database,nosql,document-store,full-text-search,log-structured
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.10
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Topic :: Database
18
+ Classifier: Topic :: Database :: Database Engines/Servers
19
+ Requires-Python: >=3.10
20
+ Description-Content-Type: text/markdown
21
+ Requires-Dist: colorama>=0.4.6
22
+ Requires-Dist: reedsolo>=1.7.0
23
+
24
+ # MkDB
25
+
26
+ MkDB is a Custom Log-Structured Merge & Partitioned Redundant Storage Engine built entirely in Python. It provides a robust, highly-available NoSQL/Document database experience with secondary indexing, full-text search, and dual-protocol network access.
27
+
28
+ ## Architecture Highlights
29
+
30
+ - **Rolling Log Storage Engine**: Append-only storage format guaranteeing high write availability with safe background compaction.
31
+ - **RAM Cache & Debounced Write Queue**: In-memory caching and debounced batching for extreme performance under high write load.
32
+ - **Query Engine & Secondary Indexes**: Fully featured query evaluation including numeric range checks and tokenized full-text inverted indexes.
33
+ - **Data Integrity**: Multi-disk mirroring and Reed-Solomon parity encoding for proactive self-healing and failover.
34
+ - **Dual Protocols**: Accessible via high-speed, persistent TCP WebSockets or standard stateless REST HTTP endpoints.
35
+ - **Web Administration UI**: Includes an embedded web control panel out-of-the-box (`/control` endpoint).
36
+
37
+ ## Project Structure
38
+
39
+ - `src/db/`: The core database engine (storage primitives, RAM caching, query evaluator, auto-compaction and parity management).
40
+ - `src/server/`: The networking boundary. Houses the TCP Socket and HTTP REST servers, as well as the web-based Control Panel.
41
+ - `src/config/`: Configuration schemas for tailoring memory limits, storage thresholds, and cluster layout.
42
+ - `sdk/`: The official `MkDBClient` for programmatic interaction from Python code.
43
+
44
+ ## Getting Started
45
+
46
+ ### Prerequisites
47
+ - Python 3.10+
48
+ - The database storage format is built into MkDB natively, but you'll need the following for advanced data integrity features (Reed-Solomon logic):
49
+ ```bash
50
+ pip install reedsolo
51
+ ```
52
+
53
+ ### Starting the Server
54
+ MkDB operates as a CLI tool. Launch the engine by pointing it to your desired database directory (which must contain a `config.json` file configuring your stores and network bindings):
55
+ ```bash
56
+ mkdb /path/to/your/db
57
+ ```
58
+ Once running, the database will host both TCP socket and HTTP interfaces as specified in your `config.json`. The web control panel is accessible via your browser (check server output for the bound port, normally `http://localhost:<port>`).
59
+
60
+ ### Using the Python SDK
61
+ The `MkDBClient` connects seamlessly to your database and abstracts the dual-protocol system:
62
+
63
+ ```python
64
+ from sdk.mkdb_client import MkDBClient
65
+
66
+ client = MkDBClient()
67
+ client.connect(host="127.0.0.1", port=8080)
68
+
69
+ # Writing a document (computes delta updates intelligently)
70
+ client.set(
71
+ store="products",
72
+ record_id="prod_001",
73
+ data={"name": "Steel Bolt", "price": 9.99, "category": "fasteners"}
74
+ )
75
+
76
+ # Reading a document
77
+ record = client.get("products", "prod_001")
78
+
79
+ # Querying with filters
80
+ results = client.query("products", filter={
81
+ "price": {"<=": 10.00},
82
+ "category": ["fasteners"]
83
+ })
84
+ ```
85
+
86
+ See the `docs/` folder for comprehensive guides on the Query Syntax and SDK Reference.
@@ -0,0 +1,57 @@
1
+ README.md
2
+ pyproject.toml
3
+ PyMkDB.egg-info/PKG-INFO
4
+ PyMkDB.egg-info/SOURCES.txt
5
+ PyMkDB.egg-info/dependency_links.txt
6
+ PyMkDB.egg-info/entry_points.txt
7
+ PyMkDB.egg-info/requires.txt
8
+ PyMkDB.egg-info/top_level.txt
9
+ pymkdb/__init__.py
10
+ pymkdb/cli.py
11
+ sdk/__init__.py
12
+ sdk/connection.py
13
+ sdk/delta.py
14
+ sdk/http_connection.py
15
+ sdk/mkdb_client.py
16
+ sdk/responses.py
17
+ src/__init__.py
18
+ src/config/db.py
19
+ src/config/server.py
20
+ src/db/__init__.py
21
+ src/db/cache/__init__.py
22
+ src/db/cache/ram_cache.py
23
+ src/db/cache/write_queue.py
24
+ src/db/maintenance/__init__.py
25
+ src/db/maintenance/compactor.py
26
+ src/db/maintenance/task_scheduler.py
27
+ src/db/objects/store.py
28
+ src/db/parity/__init__.py
29
+ src/db/parity/parity_manager.py
30
+ src/db/query/__init__.py
31
+ src/db/query/full_text_index.py
32
+ src/db/query/numeric_index.py
33
+ src/db/query/query_engine.py
34
+ src/db/query/tokenizer.py
35
+ src/db/query_workers/__init__.py
36
+ src/db/query_workers/dispatcher.py
37
+ src/db/query_workers/task.py
38
+ src/db/query_workers/worker.py
39
+ src/db/requesting/main.py
40
+ src/db/storage/__init__.py
41
+ src/db/storage/blob_store.py
42
+ src/db/storage/index_manager.py
43
+ src/db/storage/log_manager.py
44
+ src/db/storage/serializer.py
45
+ src/filing/__init__.py
46
+ src/objects/__init__.py
47
+ src/runtime/__init__.py
48
+ src/server/__init__.py
49
+ src/server/event_log.py
50
+ src/server/coms/actions.py
51
+ src/server/coms/http.py
52
+ src/server/coms/http_handlers.py
53
+ src/server/coms/metrics.py
54
+ src/server/coms/socket.py
55
+ src/server/coms/socket_protocol.py
56
+ src/server/control/server.py
57
+ src/server/control/api/actions.py
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ mkdb = pymkdb.cli:main
@@ -0,0 +1,2 @@
1
+ colorama>=0.4.6
2
+ reedsolo>=1.7.0
@@ -0,0 +1,3 @@
1
+ pymkdb
2
+ sdk
3
+ src
pymkdb-0.1.0/README.md ADDED
@@ -0,0 +1,63 @@
1
+ # MkDB
2
+
3
+ MkDB is a Custom Log-Structured Merge & Partitioned Redundant Storage Engine built entirely in Python. It provides a robust, highly-available NoSQL/Document database experience with secondary indexing, full-text search, and dual-protocol network access.
4
+
5
+ ## Architecture Highlights
6
+
7
+ - **Rolling Log Storage Engine**: Append-only storage format guaranteeing high write availability with safe background compaction.
8
+ - **RAM Cache & Debounced Write Queue**: In-memory caching and debounced batching for extreme performance under high write load.
9
+ - **Query Engine & Secondary Indexes**: Fully featured query evaluation including numeric range checks and tokenized full-text inverted indexes.
10
+ - **Data Integrity**: Multi-disk mirroring and Reed-Solomon parity encoding for proactive self-healing and failover.
11
+ - **Dual Protocols**: Accessible via high-speed, persistent TCP WebSockets or standard stateless REST HTTP endpoints.
12
+ - **Web Administration UI**: Includes an embedded web control panel out-of-the-box (`/control` endpoint).
13
+
14
+ ## Project Structure
15
+
16
+ - `src/db/`: The core database engine (storage primitives, RAM caching, query evaluator, auto-compaction and parity management).
17
+ - `src/server/`: The networking boundary. Houses the TCP Socket and HTTP REST servers, as well as the web-based Control Panel.
18
+ - `src/config/`: Configuration schemas for tailoring memory limits, storage thresholds, and cluster layout.
19
+ - `sdk/`: The official `MkDBClient` for programmatic interaction from Python code.
20
+
21
+ ## Getting Started
22
+
23
+ ### Prerequisites
24
+ - Python 3.10+
25
+ - The database storage format is built into MkDB natively, but you'll need the following for advanced data integrity features (Reed-Solomon logic):
26
+ ```bash
27
+ pip install reedsolo
28
+ ```
29
+
30
+ ### Starting the Server
31
+ MkDB operates as a CLI tool. Launch the engine by pointing it to your desired database directory (which must contain a `config.json` file configuring your stores and network bindings):
32
+ ```bash
33
+ mkdb /path/to/your/db
34
+ ```
35
+ Once running, the database will host both TCP socket and HTTP interfaces as specified in your `config.json`. The web control panel is accessible via your browser (check server output for the bound port, normally `http://localhost:<port>`).
36
+
37
+ ### Using the Python SDK
38
+ The `MkDBClient` connects seamlessly to your database and abstracts the dual-protocol system:
39
+
40
+ ```python
41
+ from sdk.mkdb_client import MkDBClient
42
+
43
+ client = MkDBClient()
44
+ client.connect(host="127.0.0.1", port=8080)
45
+
46
+ # Writing a document (computes delta updates intelligently)
47
+ client.set(
48
+ store="products",
49
+ record_id="prod_001",
50
+ data={"name": "Steel Bolt", "price": 9.99, "category": "fasteners"}
51
+ )
52
+
53
+ # Reading a document
54
+ record = client.get("products", "prod_001")
55
+
56
+ # Querying with filters
57
+ results = client.query("products", filter={
58
+ "price": {"<=": 10.00},
59
+ "category": ["fasteners"]
60
+ })
61
+ ```
62
+
63
+ See the `docs/` folder for comprehensive guides on the Query Syntax and SDK Reference.
@@ -0,0 +1,6 @@
1
+ """
2
+ PyMkDB — Python client SDK and server package for MkDB.
3
+ """
4
+
5
+ __version__ = "0.1.0"
6
+ __author__ = "MNG"
@@ -0,0 +1,57 @@
1
+ """
2
+ pymkdb.cli — console entry point for the `mkdb` command.
3
+
4
+ Usage:
5
+ mkdb [PATH_TO_DB] Start the server pointing at a database directory
6
+ mkdb [PATH_TO_DB] -c Generate a default config.json in that directory
7
+ """
8
+
9
+ import os
10
+ import sys
11
+
12
+
13
+ def main():
14
+ from colorama import Fore
15
+ from src.db import mkdb
16
+ from src.config.db import mkdb_config
17
+ from src.filing import read_json, write_json
18
+ from src.runtime import runtime_settings
19
+
20
+ print(f"""{Fore.CYAN}
21
+ ╔═══════════════════════════════════════╗
22
+ ║ ║
23
+ ║ Initializing ║
24
+ ║ MkDB ║
25
+ ║ ║
26
+ ╚═══════════════════════════════════════╝
27
+ {Fore.RESET}""")
28
+
29
+ try:
30
+ print("Initialized CWD:", os.getcwd())
31
+ pointer = sys.argv[1] if len(sys.argv) > 1 else runtime_settings.args.config
32
+ print("Pointing to:", pointer)
33
+ print(f"{Fore.CYAN}Reading Config...{Fore.RESET}")
34
+ if not pointer.endswith(".json"):
35
+ os.chdir(pointer)
36
+ CONFIG = read_json(runtime_settings.args.config)
37
+ except FileNotFoundError:
38
+ CONFIG = {}
39
+ if "-c" in sys.argv or "--generate-config" in sys.argv:
40
+ print(f"{Fore.GREEN}Generating default config file at "
41
+ f"{runtime_settings.args.config}{Fore.RESET}")
42
+ write_json(runtime_settings.args.config, mkdb_config().json)
43
+ sys.exit(0)
44
+ print(f"{Fore.YELLOW}Config file not found at {runtime_settings.args.config}")
45
+ print(f"{Fore.BLUE}Use a path to a db directory or -c to generate a "
46
+ f"new config.{Fore.RESET}")
47
+ sys.exit(1)
48
+ except Exception as e:
49
+ print(f"{Fore.RED}Error loading config: {e}{Fore.RESET}")
50
+ sys.exit(1)
51
+
52
+ DATA_BASE = mkdb(CONFIG)
53
+ DATA_BASE.run()
54
+
55
+
56
+ if __name__ == "__main__":
57
+ main()
@@ -0,0 +1,40 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "PyMkDB"
7
+ version = "0.1.0"
8
+ description = "A log-structured, partitioned NoSQL database engine with full-text search, numeric indexes, and dual TCP/HTTP protocols."
9
+ readme = "README.md"
10
+ license = { text = "MIT" }
11
+ requires-python = ">=3.10"
12
+ authors = [
13
+ { name = "MNG" }
14
+ ]
15
+ keywords = ["database", "nosql", "document-store", "full-text-search", "log-structured"]
16
+ classifiers = [
17
+ "Development Status :: 3 - Alpha",
18
+ "Intended Audience :: Developers",
19
+ "License :: OSI Approved :: MIT License",
20
+ "Programming Language :: Python :: 3",
21
+ "Programming Language :: Python :: 3.10",
22
+ "Programming Language :: Python :: 3.11",
23
+ "Programming Language :: Python :: 3.12",
24
+ "Topic :: Database",
25
+ "Topic :: Database :: Database Engines/Servers",
26
+ ]
27
+ dependencies = [
28
+ "colorama>=0.4.6",
29
+ "reedsolo>=1.7.0",
30
+ ]
31
+
32
+ [project.scripts]
33
+ mkdb = "pymkdb.cli:main"
34
+
35
+ [project.urls]
36
+ Homepage = "https://github.com/MNG/MkDB"
37
+ Repository = "https://github.com/MNG/MkDB"
38
+
39
+ [tool.setuptools.packages.find]
40
+ include = ["pymkdb*", "src*", "sdk*"]
@@ -0,0 +1 @@
1
+ # MkDB SDK
@@ -0,0 +1,225 @@
1
+ """
2
+ Low-level persistent TCP connection to a MkDB socket server.
3
+
4
+ Wire format: 4-byte big-endian uint32 length + UTF-8 JSON payload.
5
+ Mirrors src/server/coms/socket_protocol.py (client-side copy).
6
+ """
7
+
8
+ import json
9
+ import socket
10
+ import struct
11
+ import threading
12
+ import uuid
13
+ from typing import Callable, Optional
14
+
15
+
16
+ MAX_FRAME = 10 * 1024 * 1024 # 10 MB
17
+
18
+
19
+ def _recv_exact(sock: socket.socket, n: int) -> bytes:
20
+ buf = b""
21
+ while len(buf) < n:
22
+ chunk = sock.recv(n - len(buf))
23
+ if not chunk:
24
+ raise ConnectionError("Connection closed")
25
+ buf += chunk
26
+ return buf
27
+
28
+
29
+ def _encode(payload: dict) -> bytes:
30
+ body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
31
+ return struct.pack(">I", len(body)) + body
32
+
33
+
34
+ def _decode(data: bytes) -> dict:
35
+ return json.loads(data.decode("utf-8"))
36
+
37
+
38
+ class Connection:
39
+ """
40
+ Thread-safe persistent connection to MkDB.
41
+
42
+ Outbound: _send() serialises and writes to socket.
43
+ Inbound: background _reader_loop() dispatches to registered handlers.
44
+ """
45
+
46
+ def __init__(self, host: str = "127.0.0.1", port: int = 9001,
47
+ recv_timeout: float = 30.0,
48
+ access: str = "R",
49
+ username: str = "",
50
+ password: str = ""):
51
+ self.host = host
52
+ self.port = port
53
+ self.recv_timeout = recv_timeout
54
+ self._access = access.upper() if access.upper() in ("R", "W", "RW") else "R"
55
+ self._username = username
56
+ self._password = password
57
+ self._sock: Optional[socket.socket] = None
58
+ self._lock = threading.Lock()
59
+ self._pending: dict[str, threading.Event] = {} # correlation_id -> Event
60
+ self._results: dict[str, dict] = {} # correlation_id -> response dict
61
+ self._push_handlers: list[Callable[[dict], None]] = [] # for server-push events
62
+ self._running = False
63
+ self.can_read = False
64
+ self.can_write = False
65
+
66
+ # ------------------------------------------------------------------
67
+ # Connect / disconnect
68
+ # ------------------------------------------------------------------
69
+
70
+ def connect(self) -> None:
71
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
72
+ sock.settimeout(self.recv_timeout)
73
+ sock.connect((self.host, self.port))
74
+
75
+ # ── Handshake ────────────────────────────────────────────────
76
+ perm = self._handshake_recv(sock)
77
+ if perm.get("type") != "permissions":
78
+ sock.close()
79
+ raise ConnectionError(f"Expected 'permissions', got: {perm}")
80
+
81
+ read_protected = perm.get("read_protected", False)
82
+ write_protected = perm.get("write_protected", False)
83
+
84
+ # Declare desired access level
85
+ self._handshake_send(sock, {"access": self._access})
86
+
87
+ need_auth = (
88
+ (self._access in ("W", "RW") and write_protected)
89
+ or (self._access == "R" and read_protected)
90
+ )
91
+
92
+ if need_auth:
93
+ challenge = self._handshake_recv(sock)
94
+ if challenge.get("type") != "auth_required":
95
+ sock.close()
96
+ raise ConnectionError(f"Expected 'auth_required', got: {challenge}")
97
+ if not self._password:
98
+ sock.close()
99
+ raise PermissionError("Server requires authentication but no password provided")
100
+ self._handshake_send(sock, {
101
+ "type": "auth",
102
+ "username": self._username,
103
+ "password": self._password,
104
+ })
105
+ result = self._handshake_recv(sock)
106
+ if result.get("type") == "error":
107
+ sock.close()
108
+ raise PermissionError(result.get("error", "Authentication failed"))
109
+ if result.get("type") != "auth_ok":
110
+ sock.close()
111
+ raise ConnectionError(f"Unexpected handshake response: {result}")
112
+ self.can_read = result.get("can_read", False)
113
+ self.can_write = result.get("can_write", False)
114
+ else:
115
+ ready = self._handshake_recv(sock)
116
+ if ready.get("type") == "error":
117
+ sock.close()
118
+ raise ConnectionError(ready.get("error", "Connection refused"))
119
+ self.can_read = ready.get("can_read", True)
120
+ self.can_write = ready.get("can_write", False)
121
+
122
+ self._sock = sock
123
+ self._running = True
124
+ reader = threading.Thread(
125
+ target=self._reader_loop, daemon=True, name="MkDB-SDK-reader"
126
+ )
127
+ reader.start()
128
+
129
+ @staticmethod
130
+ def _handshake_send(sock: socket.socket, msg: dict) -> None:
131
+ body = json.dumps(msg).encode("utf-8")
132
+ sock.sendall(struct.pack(">I", len(body)) + body)
133
+
134
+ @staticmethod
135
+ def _handshake_recv(sock: socket.socket) -> dict:
136
+ hdr = _recv_exact(sock, 4)
137
+ length = struct.unpack(">I", hdr)[0]
138
+ return json.loads(_recv_exact(sock, length).decode("utf-8"))
139
+
140
+ def close(self) -> None:
141
+ self._running = False
142
+ if self._sock:
143
+ try:
144
+ self._sock.close()
145
+ except Exception:
146
+ pass
147
+ self._sock = None
148
+
149
+ # ------------------------------------------------------------------
150
+ # Send / receive
151
+ # ------------------------------------------------------------------
152
+
153
+ def send(self, payload: dict) -> dict:
154
+ """
155
+ Send a request and block until the matching response arrives.
156
+ Returns the response dict.
157
+ """
158
+ correlation_id = str(uuid.uuid4())
159
+ payload["id"] = correlation_id
160
+ payload.setdefault("type", "request")
161
+
162
+ event = threading.Event()
163
+ with self._lock:
164
+ self._pending[correlation_id] = event
165
+
166
+ frame = _encode(payload)
167
+ with self._lock:
168
+ self._sock.sendall(frame)
169
+
170
+ event.wait(timeout=self.recv_timeout)
171
+ with self._lock:
172
+ result = self._results.pop(correlation_id, None)
173
+ self._pending.pop(correlation_id, None)
174
+ if result is None:
175
+ raise TimeoutError("No response received within timeout")
176
+ return result
177
+
178
+ def send_raw(self, payload: dict) -> None:
179
+ """Fire-and-forget (used for subscribe)."""
180
+ frame = _encode(payload)
181
+ with self._lock:
182
+ self._sock.sendall(frame)
183
+
184
+ def register_push_handler(self, handler: Callable[[dict], None]) -> None:
185
+ """Register a callback for server-push (ping, update, subscribed, disconnect)."""
186
+ self._push_handlers.append(handler)
187
+
188
+ # ------------------------------------------------------------------
189
+ # Reader loop (background thread)
190
+ # ------------------------------------------------------------------
191
+
192
+ def _reader_loop(self) -> None:
193
+ while self._running and self._sock:
194
+ try:
195
+ length_bytes = _recv_exact(self._sock, 4)
196
+ length = struct.unpack(">I", length_bytes)[0]
197
+ if length > MAX_FRAME:
198
+ break # protocol violation — disconnect
199
+ payload_bytes = _recv_exact(self._sock, length)
200
+ msg = _decode(payload_bytes)
201
+ except Exception:
202
+ break
203
+
204
+ msg_type = msg.get("type")
205
+ correlation_id = msg.get("id")
206
+
207
+ if msg_type == "response" and correlation_id:
208
+ with self._lock:
209
+ event = self._pending.get(correlation_id)
210
+ if event:
211
+ self._results[correlation_id] = msg
212
+ event.set()
213
+ elif msg_type == "ping":
214
+ # Respond with pong
215
+ try:
216
+ self._sock.sendall(_encode({"type": "pong"}))
217
+ except Exception:
218
+ break
219
+ else:
220
+ # Server-push: dispatch to registered handlers
221
+ for handler in self._push_handlers:
222
+ try:
223
+ handler(msg)
224
+ except Exception:
225
+ pass
@@ -0,0 +1,19 @@
1
+ """
2
+ Delta helpers — flatten nested dicts into dot-notation keys for MkDB writes.
3
+
4
+ Example:
5
+ flatten({"product": {"name": "Widget", "price": 9.99}})
6
+ # → {"product.name": "Widget", "product.price": 9.99}
7
+ """
8
+
9
+
10
+ def flatten(d: dict, prefix: str = "", sep: str = ".") -> dict:
11
+ """Recursively flatten a nested dict into dot-notation keys."""
12
+ result = {}
13
+ for key, value in d.items():
14
+ full_key = f"{prefix}{sep}{key}" if prefix else key
15
+ if isinstance(value, dict):
16
+ result.update(flatten(value, full_key, sep))
17
+ else:
18
+ result[full_key] = value
19
+ return result