PyMkDB 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pymkdb-0.1.0/PKG-INFO +86 -0
- pymkdb-0.1.0/PyMkDB.egg-info/PKG-INFO +86 -0
- pymkdb-0.1.0/PyMkDB.egg-info/SOURCES.txt +57 -0
- pymkdb-0.1.0/PyMkDB.egg-info/dependency_links.txt +1 -0
- pymkdb-0.1.0/PyMkDB.egg-info/entry_points.txt +2 -0
- pymkdb-0.1.0/PyMkDB.egg-info/requires.txt +2 -0
- pymkdb-0.1.0/PyMkDB.egg-info/top_level.txt +3 -0
- pymkdb-0.1.0/README.md +63 -0
- pymkdb-0.1.0/pymkdb/__init__.py +6 -0
- pymkdb-0.1.0/pymkdb/cli.py +57 -0
- pymkdb-0.1.0/pyproject.toml +40 -0
- pymkdb-0.1.0/sdk/__init__.py +1 -0
- pymkdb-0.1.0/sdk/connection.py +225 -0
- pymkdb-0.1.0/sdk/delta.py +19 -0
- pymkdb-0.1.0/sdk/http_connection.py +180 -0
- pymkdb-0.1.0/sdk/mkdb_client.py +226 -0
- pymkdb-0.1.0/sdk/responses.py +154 -0
- pymkdb-0.1.0/setup.cfg +4 -0
- pymkdb-0.1.0/src/__init__.py +1 -0
- pymkdb-0.1.0/src/config/db.py +227 -0
- pymkdb-0.1.0/src/config/server.py +52 -0
- pymkdb-0.1.0/src/db/__init__.py +207 -0
- pymkdb-0.1.0/src/db/cache/__init__.py +1 -0
- pymkdb-0.1.0/src/db/cache/ram_cache.py +144 -0
- pymkdb-0.1.0/src/db/cache/write_queue.py +156 -0
- pymkdb-0.1.0/src/db/maintenance/__init__.py +0 -0
- pymkdb-0.1.0/src/db/maintenance/compactor.py +118 -0
- pymkdb-0.1.0/src/db/maintenance/task_scheduler.py +73 -0
- pymkdb-0.1.0/src/db/objects/store.py +283 -0
- pymkdb-0.1.0/src/db/parity/__init__.py +0 -0
- pymkdb-0.1.0/src/db/parity/parity_manager.py +196 -0
- pymkdb-0.1.0/src/db/query/__init__.py +1 -0
- pymkdb-0.1.0/src/db/query/full_text_index.py +168 -0
- pymkdb-0.1.0/src/db/query/numeric_index.py +196 -0
- pymkdb-0.1.0/src/db/query/query_engine.py +308 -0
- pymkdb-0.1.0/src/db/query/tokenizer.py +48 -0
- pymkdb-0.1.0/src/db/query_workers/__init__.py +16 -0
- pymkdb-0.1.0/src/db/query_workers/dispatcher.py +339 -0
- pymkdb-0.1.0/src/db/query_workers/task.py +78 -0
- pymkdb-0.1.0/src/db/query_workers/worker.py +292 -0
- pymkdb-0.1.0/src/db/requesting/main.py +0 -0
- pymkdb-0.1.0/src/db/storage/__init__.py +1 -0
- pymkdb-0.1.0/src/db/storage/blob_store.py +47 -0
- pymkdb-0.1.0/src/db/storage/index_manager.py +92 -0
- pymkdb-0.1.0/src/db/storage/log_manager.py +119 -0
- pymkdb-0.1.0/src/db/storage/serializer.py +38 -0
- pymkdb-0.1.0/src/filing/__init__.py +31 -0
- pymkdb-0.1.0/src/objects/__init__.py +190 -0
- pymkdb-0.1.0/src/runtime/__init__.py +15 -0
- pymkdb-0.1.0/src/server/__init__.py +0 -0
- pymkdb-0.1.0/src/server/coms/actions.py +209 -0
- pymkdb-0.1.0/src/server/coms/http.py +46 -0
- pymkdb-0.1.0/src/server/coms/http_handlers.py +445 -0
- pymkdb-0.1.0/src/server/coms/metrics.py +231 -0
- pymkdb-0.1.0/src/server/coms/socket.py +461 -0
- pymkdb-0.1.0/src/server/coms/socket_protocol.py +54 -0
- pymkdb-0.1.0/src/server/control/api/actions.py +1001 -0
- pymkdb-0.1.0/src/server/control/server.py +404 -0
- pymkdb-0.1.0/src/server/event_log.py +58 -0
pymkdb-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: PyMkDB
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A log-structured, partitioned NoSQL database engine with full-text search, numeric indexes, and dual TCP/HTTP protocols.
|
|
5
|
+
Author: MNG
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/MNG/MkDB
|
|
8
|
+
Project-URL: Repository, https://github.com/MNG/MkDB
|
|
9
|
+
Keywords: database,nosql,document-store,full-text-search,log-structured
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Database
|
|
18
|
+
Classifier: Topic :: Database :: Database Engines/Servers
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
Requires-Dist: colorama>=0.4.6
|
|
22
|
+
Requires-Dist: reedsolo>=1.7.0
|
|
23
|
+
|
|
24
|
+
# MkDB
|
|
25
|
+
|
|
26
|
+
MkDB is a Custom Log-Structured Merge & Partitioned Redundant Storage Engine built entirely in Python. It provides a robust, highly-available NoSQL/Document database experience with secondary indexing, full-text search, and dual-protocol network access.
|
|
27
|
+
|
|
28
|
+
## Architecture Highlights
|
|
29
|
+
|
|
30
|
+
- **Rolling Log Storage Engine**: Append-only storage format guaranteeing high write availability with safe background compaction.
|
|
31
|
+
- **RAM Cache & Debounced Write Queue**: In-memory caching and debounced batching for extreme performance under high write load.
|
|
32
|
+
- **Query Engine & Secondary Indexes**: Fully featured query evaluation including numeric range checks and tokenized full-text inverted indexes.
|
|
33
|
+
- **Data Integrity**: Multi-disk mirroring and Reed-Solomon parity encoding for proactive self-healing and failover.
|
|
34
|
+
- **Dual Protocols**: Accessible via high-speed, persistent TCP WebSockets or standard stateless REST HTTP endpoints.
|
|
35
|
+
- **Web Administration UI**: Includes an embedded web control panel out-of-the-box (`/control` endpoint).
|
|
36
|
+
|
|
37
|
+
## Project Structure
|
|
38
|
+
|
|
39
|
+
- `src/db/`: The core database engine (storage primitives, RAM caching, query evaluator, auto-compaction and parity management).
|
|
40
|
+
- `src/server/`: The networking boundary. Houses the TCP Socket and HTTP REST servers, as well as the web-based Control Panel.
|
|
41
|
+
- `src/config/`: Configuration schemas for tailoring memory limits, storage thresholds, and cluster layout.
|
|
42
|
+
- `sdk/`: The official `MkDBClient` for programmatic interaction from Python code.
|
|
43
|
+
|
|
44
|
+
## Getting Started
|
|
45
|
+
|
|
46
|
+
### Prerequisites
|
|
47
|
+
- Python 3.10+
|
|
48
|
+
- The database storage format is built into MkDB natively, but you'll need the following for advanced data integrity features (Reed-Solomon logic):
|
|
49
|
+
```bash
|
|
50
|
+
pip install reedsolo
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### Starting the Server
|
|
54
|
+
MkDB operates as a CLI tool. Launch the engine by pointing it to your desired database directory (which must contain a `config.json` file configuring your stores and network bindings):
|
|
55
|
+
```bash
|
|
56
|
+
mkdb /path/to/your/db
|
|
57
|
+
```
|
|
58
|
+
Once running, the database will host both TCP socket and HTTP interfaces as specified in your `config.json`. The web control panel is accessible via your browser (check server output for the bound port, normally `http://localhost:<port>`).
|
|
59
|
+
|
|
60
|
+
### Using the Python SDK
|
|
61
|
+
The `MkDBClient` connects seamlessly to your database and abstracts the dual-protocol system:
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
from sdk.mkdb_client import MkDBClient
|
|
65
|
+
|
|
66
|
+
client = MkDBClient()
|
|
67
|
+
client.connect(host="127.0.0.1", port=8080)
|
|
68
|
+
|
|
69
|
+
# Writing a document (computes delta updates intelligently)
|
|
70
|
+
client.set(
|
|
71
|
+
store="products",
|
|
72
|
+
record_id="prod_001",
|
|
73
|
+
data={"name": "Steel Bolt", "price": 9.99, "category": "fasteners"}
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
# Reading a document
|
|
77
|
+
record = client.get("products", "prod_001")
|
|
78
|
+
|
|
79
|
+
# Querying with filters
|
|
80
|
+
results = client.query("products", filter={
|
|
81
|
+
"price": {"<=": 10.00},
|
|
82
|
+
"category": ["fasteners"]
|
|
83
|
+
})
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
See the `docs/` folder for comprehensive guides on the Query Syntax and SDK Reference.
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: PyMkDB
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A log-structured, partitioned NoSQL database engine with full-text search, numeric indexes, and dual TCP/HTTP protocols.
|
|
5
|
+
Author: MNG
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/MNG/MkDB
|
|
8
|
+
Project-URL: Repository, https://github.com/MNG/MkDB
|
|
9
|
+
Keywords: database,nosql,document-store,full-text-search,log-structured
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Database
|
|
18
|
+
Classifier: Topic :: Database :: Database Engines/Servers
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
Requires-Dist: colorama>=0.4.6
|
|
22
|
+
Requires-Dist: reedsolo>=1.7.0
|
|
23
|
+
|
|
24
|
+
# MkDB
|
|
25
|
+
|
|
26
|
+
MkDB is a Custom Log-Structured Merge & Partitioned Redundant Storage Engine built entirely in Python. It provides a robust, highly-available NoSQL/Document database experience with secondary indexing, full-text search, and dual-protocol network access.
|
|
27
|
+
|
|
28
|
+
## Architecture Highlights
|
|
29
|
+
|
|
30
|
+
- **Rolling Log Storage Engine**: Append-only storage format guaranteeing high write availability with safe background compaction.
|
|
31
|
+
- **RAM Cache & Debounced Write Queue**: In-memory caching and debounced batching for extreme performance under high write load.
|
|
32
|
+
- **Query Engine & Secondary Indexes**: Fully featured query evaluation including numeric range checks and tokenized full-text inverted indexes.
|
|
33
|
+
- **Data Integrity**: Multi-disk mirroring and Reed-Solomon parity encoding for proactive self-healing and failover.
|
|
34
|
+
- **Dual Protocols**: Accessible via high-speed, persistent TCP WebSockets or standard stateless REST HTTP endpoints.
|
|
35
|
+
- **Web Administration UI**: Includes an embedded web control panel out-of-the-box (`/control` endpoint).
|
|
36
|
+
|
|
37
|
+
## Project Structure
|
|
38
|
+
|
|
39
|
+
- `src/db/`: The core database engine (storage primitives, RAM caching, query evaluator, auto-compaction and parity management).
|
|
40
|
+
- `src/server/`: The networking boundary. Houses the TCP Socket and HTTP REST servers, as well as the web-based Control Panel.
|
|
41
|
+
- `src/config/`: Configuration schemas for tailoring memory limits, storage thresholds, and cluster layout.
|
|
42
|
+
- `sdk/`: The official `MkDBClient` for programmatic interaction from Python code.
|
|
43
|
+
|
|
44
|
+
## Getting Started
|
|
45
|
+
|
|
46
|
+
### Prerequisites
|
|
47
|
+
- Python 3.10+
|
|
48
|
+
- The database storage format is built into MkDB natively, but you'll need the following for advanced data integrity features (Reed-Solomon logic):
|
|
49
|
+
```bash
|
|
50
|
+
pip install reedsolo
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### Starting the Server
|
|
54
|
+
MkDB operates as a CLI tool. Launch the engine by pointing it to your desired database directory (which must contain a `config.json` file configuring your stores and network bindings):
|
|
55
|
+
```bash
|
|
56
|
+
mkdb /path/to/your/db
|
|
57
|
+
```
|
|
58
|
+
Once running, the database will host both TCP socket and HTTP interfaces as specified in your `config.json`. The web control panel is accessible via your browser (check server output for the bound port, normally `http://localhost:<port>`).
|
|
59
|
+
|
|
60
|
+
### Using the Python SDK
|
|
61
|
+
The `MkDBClient` connects seamlessly to your database and abstracts the dual-protocol system:
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
from sdk.mkdb_client import MkDBClient
|
|
65
|
+
|
|
66
|
+
client = MkDBClient()
|
|
67
|
+
client.connect(host="127.0.0.1", port=8080)
|
|
68
|
+
|
|
69
|
+
# Writing a document (computes delta updates intelligently)
|
|
70
|
+
client.set(
|
|
71
|
+
store="products",
|
|
72
|
+
record_id="prod_001",
|
|
73
|
+
data={"name": "Steel Bolt", "price": 9.99, "category": "fasteners"}
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
# Reading a document
|
|
77
|
+
record = client.get("products", "prod_001")
|
|
78
|
+
|
|
79
|
+
# Querying with filters
|
|
80
|
+
results = client.query("products", filter={
|
|
81
|
+
"price": {"<=": 10.00},
|
|
82
|
+
"category": ["fasteners"]
|
|
83
|
+
})
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
See the `docs/` folder for comprehensive guides on the Query Syntax and SDK Reference.
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
PyMkDB.egg-info/PKG-INFO
|
|
4
|
+
PyMkDB.egg-info/SOURCES.txt
|
|
5
|
+
PyMkDB.egg-info/dependency_links.txt
|
|
6
|
+
PyMkDB.egg-info/entry_points.txt
|
|
7
|
+
PyMkDB.egg-info/requires.txt
|
|
8
|
+
PyMkDB.egg-info/top_level.txt
|
|
9
|
+
pymkdb/__init__.py
|
|
10
|
+
pymkdb/cli.py
|
|
11
|
+
sdk/__init__.py
|
|
12
|
+
sdk/connection.py
|
|
13
|
+
sdk/delta.py
|
|
14
|
+
sdk/http_connection.py
|
|
15
|
+
sdk/mkdb_client.py
|
|
16
|
+
sdk/responses.py
|
|
17
|
+
src/__init__.py
|
|
18
|
+
src/config/db.py
|
|
19
|
+
src/config/server.py
|
|
20
|
+
src/db/__init__.py
|
|
21
|
+
src/db/cache/__init__.py
|
|
22
|
+
src/db/cache/ram_cache.py
|
|
23
|
+
src/db/cache/write_queue.py
|
|
24
|
+
src/db/maintenance/__init__.py
|
|
25
|
+
src/db/maintenance/compactor.py
|
|
26
|
+
src/db/maintenance/task_scheduler.py
|
|
27
|
+
src/db/objects/store.py
|
|
28
|
+
src/db/parity/__init__.py
|
|
29
|
+
src/db/parity/parity_manager.py
|
|
30
|
+
src/db/query/__init__.py
|
|
31
|
+
src/db/query/full_text_index.py
|
|
32
|
+
src/db/query/numeric_index.py
|
|
33
|
+
src/db/query/query_engine.py
|
|
34
|
+
src/db/query/tokenizer.py
|
|
35
|
+
src/db/query_workers/__init__.py
|
|
36
|
+
src/db/query_workers/dispatcher.py
|
|
37
|
+
src/db/query_workers/task.py
|
|
38
|
+
src/db/query_workers/worker.py
|
|
39
|
+
src/db/requesting/main.py
|
|
40
|
+
src/db/storage/__init__.py
|
|
41
|
+
src/db/storage/blob_store.py
|
|
42
|
+
src/db/storage/index_manager.py
|
|
43
|
+
src/db/storage/log_manager.py
|
|
44
|
+
src/db/storage/serializer.py
|
|
45
|
+
src/filing/__init__.py
|
|
46
|
+
src/objects/__init__.py
|
|
47
|
+
src/runtime/__init__.py
|
|
48
|
+
src/server/__init__.py
|
|
49
|
+
src/server/event_log.py
|
|
50
|
+
src/server/coms/actions.py
|
|
51
|
+
src/server/coms/http.py
|
|
52
|
+
src/server/coms/http_handlers.py
|
|
53
|
+
src/server/coms/metrics.py
|
|
54
|
+
src/server/coms/socket.py
|
|
55
|
+
src/server/coms/socket_protocol.py
|
|
56
|
+
src/server/control/server.py
|
|
57
|
+
src/server/control/api/actions.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
pymkdb-0.1.0/README.md
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# MkDB
|
|
2
|
+
|
|
3
|
+
MkDB is a Custom Log-Structured Merge & Partitioned Redundant Storage Engine built entirely in Python. It provides a robust, highly-available NoSQL/Document database experience with secondary indexing, full-text search, and dual-protocol network access.
|
|
4
|
+
|
|
5
|
+
## Architecture Highlights
|
|
6
|
+
|
|
7
|
+
- **Rolling Log Storage Engine**: Append-only storage format guaranteeing high write availability with safe background compaction.
|
|
8
|
+
- **RAM Cache & Debounced Write Queue**: In-memory caching and debounced batching for extreme performance under high write load.
|
|
9
|
+
- **Query Engine & Secondary Indexes**: Fully featured query evaluation including numeric range checks and tokenized full-text inverted indexes.
|
|
10
|
+
- **Data Integrity**: Multi-disk mirroring and Reed-Solomon parity encoding for proactive self-healing and failover.
|
|
11
|
+
- **Dual Protocols**: Accessible via high-speed, persistent TCP WebSockets or standard stateless REST HTTP endpoints.
|
|
12
|
+
- **Web Administration UI**: Includes an embedded web control panel out-of-the-box (`/control` endpoint).
|
|
13
|
+
|
|
14
|
+
## Project Structure
|
|
15
|
+
|
|
16
|
+
- `src/db/`: The core database engine (storage primitives, RAM caching, query evaluator, auto-compaction and parity management).
|
|
17
|
+
- `src/server/`: The networking boundary. Houses the TCP Socket and HTTP REST servers, as well as the web-based Control Panel.
|
|
18
|
+
- `src/config/`: Configuration schemas for tailoring memory limits, storage thresholds, and cluster layout.
|
|
19
|
+
- `sdk/`: The official `MkDBClient` for programmatic interaction from Python code.
|
|
20
|
+
|
|
21
|
+
## Getting Started
|
|
22
|
+
|
|
23
|
+
### Prerequisites
|
|
24
|
+
- Python 3.10+
|
|
25
|
+
- The database storage format is built into MkDB natively, but you'll need the following for advanced data integrity features (Reed-Solomon logic):
|
|
26
|
+
```bash
|
|
27
|
+
pip install reedsolo
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
### Starting the Server
|
|
31
|
+
MkDB operates as a CLI tool. Launch the engine by pointing it to your desired database directory (which must contain a `config.json` file configuring your stores and network bindings):
|
|
32
|
+
```bash
|
|
33
|
+
mkdb /path/to/your/db
|
|
34
|
+
```
|
|
35
|
+
Once running, the database will host both TCP socket and HTTP interfaces as specified in your `config.json`. The web control panel is accessible via your browser (check server output for the bound port, normally `http://localhost:<port>`).
|
|
36
|
+
|
|
37
|
+
### Using the Python SDK
|
|
38
|
+
The `MkDBClient` connects seamlessly to your database and abstracts the dual-protocol system:
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
from sdk.mkdb_client import MkDBClient
|
|
42
|
+
|
|
43
|
+
client = MkDBClient()
|
|
44
|
+
client.connect(host="127.0.0.1", port=8080)
|
|
45
|
+
|
|
46
|
+
# Writing a document (computes delta updates intelligently)
|
|
47
|
+
client.set(
|
|
48
|
+
store="products",
|
|
49
|
+
record_id="prod_001",
|
|
50
|
+
data={"name": "Steel Bolt", "price": 9.99, "category": "fasteners"}
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
# Reading a document
|
|
54
|
+
record = client.get("products", "prod_001")
|
|
55
|
+
|
|
56
|
+
# Querying with filters
|
|
57
|
+
results = client.query("products", filter={
|
|
58
|
+
"price": {"<=": 10.00},
|
|
59
|
+
"category": ["fasteners"]
|
|
60
|
+
})
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
See the `docs/` folder for comprehensive guides on the Query Syntax and SDK Reference.
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""
|
|
2
|
+
pymkdb.cli — console entry point for the `mkdb` command.
|
|
3
|
+
|
|
4
|
+
Usage:
|
|
5
|
+
mkdb [PATH_TO_DB] Start the server pointing at a database directory
|
|
6
|
+
mkdb [PATH_TO_DB] -c Generate a default config.json in that directory
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
import sys
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def main():
|
|
14
|
+
from colorama import Fore
|
|
15
|
+
from src.db import mkdb
|
|
16
|
+
from src.config.db import mkdb_config
|
|
17
|
+
from src.filing import read_json, write_json
|
|
18
|
+
from src.runtime import runtime_settings
|
|
19
|
+
|
|
20
|
+
print(f"""{Fore.CYAN}
|
|
21
|
+
╔═══════════════════════════════════════╗
|
|
22
|
+
║ ║
|
|
23
|
+
║ Initializing ║
|
|
24
|
+
║ MkDB ║
|
|
25
|
+
║ ║
|
|
26
|
+
╚═══════════════════════════════════════╝
|
|
27
|
+
{Fore.RESET}""")
|
|
28
|
+
|
|
29
|
+
try:
|
|
30
|
+
print("Initialized CWD:", os.getcwd())
|
|
31
|
+
pointer = sys.argv[1] if len(sys.argv) > 1 else runtime_settings.args.config
|
|
32
|
+
print("Pointing to:", pointer)
|
|
33
|
+
print(f"{Fore.CYAN}Reading Config...{Fore.RESET}")
|
|
34
|
+
if not pointer.endswith(".json"):
|
|
35
|
+
os.chdir(pointer)
|
|
36
|
+
CONFIG = read_json(runtime_settings.args.config)
|
|
37
|
+
except FileNotFoundError:
|
|
38
|
+
CONFIG = {}
|
|
39
|
+
if "-c" in sys.argv or "--generate-config" in sys.argv:
|
|
40
|
+
print(f"{Fore.GREEN}Generating default config file at "
|
|
41
|
+
f"{runtime_settings.args.config}{Fore.RESET}")
|
|
42
|
+
write_json(runtime_settings.args.config, mkdb_config().json)
|
|
43
|
+
sys.exit(0)
|
|
44
|
+
print(f"{Fore.YELLOW}Config file not found at {runtime_settings.args.config}")
|
|
45
|
+
print(f"{Fore.BLUE}Use a path to a db directory or -c to generate a "
|
|
46
|
+
f"new config.{Fore.RESET}")
|
|
47
|
+
sys.exit(1)
|
|
48
|
+
except Exception as e:
|
|
49
|
+
print(f"{Fore.RED}Error loading config: {e}{Fore.RESET}")
|
|
50
|
+
sys.exit(1)
|
|
51
|
+
|
|
52
|
+
DATA_BASE = mkdb(CONFIG)
|
|
53
|
+
DATA_BASE.run()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
if __name__ == "__main__":
|
|
57
|
+
main()
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "PyMkDB"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "A log-structured, partitioned NoSQL database engine with full-text search, numeric indexes, and dual TCP/HTTP protocols."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "MNG" }
|
|
14
|
+
]
|
|
15
|
+
keywords = ["database", "nosql", "document-store", "full-text-search", "log-structured"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 3 - Alpha",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"License :: OSI Approved :: MIT License",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3.11",
|
|
23
|
+
"Programming Language :: Python :: 3.12",
|
|
24
|
+
"Topic :: Database",
|
|
25
|
+
"Topic :: Database :: Database Engines/Servers",
|
|
26
|
+
]
|
|
27
|
+
dependencies = [
|
|
28
|
+
"colorama>=0.4.6",
|
|
29
|
+
"reedsolo>=1.7.0",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[project.scripts]
|
|
33
|
+
mkdb = "pymkdb.cli:main"
|
|
34
|
+
|
|
35
|
+
[project.urls]
|
|
36
|
+
Homepage = "https://github.com/MNG/MkDB"
|
|
37
|
+
Repository = "https://github.com/MNG/MkDB"
|
|
38
|
+
|
|
39
|
+
[tool.setuptools.packages.find]
|
|
40
|
+
include = ["pymkdb*", "src*", "sdk*"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# MkDB SDK
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Low-level persistent TCP connection to a MkDB socket server.
|
|
3
|
+
|
|
4
|
+
Wire format: 4-byte big-endian uint32 length + UTF-8 JSON payload.
|
|
5
|
+
Mirrors src/server/coms/socket_protocol.py (client-side copy).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import socket
|
|
10
|
+
import struct
|
|
11
|
+
import threading
|
|
12
|
+
import uuid
|
|
13
|
+
from typing import Callable, Optional
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
MAX_FRAME = 10 * 1024 * 1024 # 10 MB
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _recv_exact(sock: socket.socket, n: int) -> bytes:
|
|
20
|
+
buf = b""
|
|
21
|
+
while len(buf) < n:
|
|
22
|
+
chunk = sock.recv(n - len(buf))
|
|
23
|
+
if not chunk:
|
|
24
|
+
raise ConnectionError("Connection closed")
|
|
25
|
+
buf += chunk
|
|
26
|
+
return buf
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _encode(payload: dict) -> bytes:
|
|
30
|
+
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
31
|
+
return struct.pack(">I", len(body)) + body
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _decode(data: bytes) -> dict:
|
|
35
|
+
return json.loads(data.decode("utf-8"))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class Connection:
|
|
39
|
+
"""
|
|
40
|
+
Thread-safe persistent connection to MkDB.
|
|
41
|
+
|
|
42
|
+
Outbound: _send() serialises and writes to socket.
|
|
43
|
+
Inbound: background _reader_loop() dispatches to registered handlers.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
def __init__(self, host: str = "127.0.0.1", port: int = 9001,
|
|
47
|
+
recv_timeout: float = 30.0,
|
|
48
|
+
access: str = "R",
|
|
49
|
+
username: str = "",
|
|
50
|
+
password: str = ""):
|
|
51
|
+
self.host = host
|
|
52
|
+
self.port = port
|
|
53
|
+
self.recv_timeout = recv_timeout
|
|
54
|
+
self._access = access.upper() if access.upper() in ("R", "W", "RW") else "R"
|
|
55
|
+
self._username = username
|
|
56
|
+
self._password = password
|
|
57
|
+
self._sock: Optional[socket.socket] = None
|
|
58
|
+
self._lock = threading.Lock()
|
|
59
|
+
self._pending: dict[str, threading.Event] = {} # correlation_id -> Event
|
|
60
|
+
self._results: dict[str, dict] = {} # correlation_id -> response dict
|
|
61
|
+
self._push_handlers: list[Callable[[dict], None]] = [] # for server-push events
|
|
62
|
+
self._running = False
|
|
63
|
+
self.can_read = False
|
|
64
|
+
self.can_write = False
|
|
65
|
+
|
|
66
|
+
# ------------------------------------------------------------------
|
|
67
|
+
# Connect / disconnect
|
|
68
|
+
# ------------------------------------------------------------------
|
|
69
|
+
|
|
70
|
+
def connect(self) -> None:
|
|
71
|
+
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
72
|
+
sock.settimeout(self.recv_timeout)
|
|
73
|
+
sock.connect((self.host, self.port))
|
|
74
|
+
|
|
75
|
+
# ── Handshake ────────────────────────────────────────────────
|
|
76
|
+
perm = self._handshake_recv(sock)
|
|
77
|
+
if perm.get("type") != "permissions":
|
|
78
|
+
sock.close()
|
|
79
|
+
raise ConnectionError(f"Expected 'permissions', got: {perm}")
|
|
80
|
+
|
|
81
|
+
read_protected = perm.get("read_protected", False)
|
|
82
|
+
write_protected = perm.get("write_protected", False)
|
|
83
|
+
|
|
84
|
+
# Declare desired access level
|
|
85
|
+
self._handshake_send(sock, {"access": self._access})
|
|
86
|
+
|
|
87
|
+
need_auth = (
|
|
88
|
+
(self._access in ("W", "RW") and write_protected)
|
|
89
|
+
or (self._access == "R" and read_protected)
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
if need_auth:
|
|
93
|
+
challenge = self._handshake_recv(sock)
|
|
94
|
+
if challenge.get("type") != "auth_required":
|
|
95
|
+
sock.close()
|
|
96
|
+
raise ConnectionError(f"Expected 'auth_required', got: {challenge}")
|
|
97
|
+
if not self._password:
|
|
98
|
+
sock.close()
|
|
99
|
+
raise PermissionError("Server requires authentication but no password provided")
|
|
100
|
+
self._handshake_send(sock, {
|
|
101
|
+
"type": "auth",
|
|
102
|
+
"username": self._username,
|
|
103
|
+
"password": self._password,
|
|
104
|
+
})
|
|
105
|
+
result = self._handshake_recv(sock)
|
|
106
|
+
if result.get("type") == "error":
|
|
107
|
+
sock.close()
|
|
108
|
+
raise PermissionError(result.get("error", "Authentication failed"))
|
|
109
|
+
if result.get("type") != "auth_ok":
|
|
110
|
+
sock.close()
|
|
111
|
+
raise ConnectionError(f"Unexpected handshake response: {result}")
|
|
112
|
+
self.can_read = result.get("can_read", False)
|
|
113
|
+
self.can_write = result.get("can_write", False)
|
|
114
|
+
else:
|
|
115
|
+
ready = self._handshake_recv(sock)
|
|
116
|
+
if ready.get("type") == "error":
|
|
117
|
+
sock.close()
|
|
118
|
+
raise ConnectionError(ready.get("error", "Connection refused"))
|
|
119
|
+
self.can_read = ready.get("can_read", True)
|
|
120
|
+
self.can_write = ready.get("can_write", False)
|
|
121
|
+
|
|
122
|
+
self._sock = sock
|
|
123
|
+
self._running = True
|
|
124
|
+
reader = threading.Thread(
|
|
125
|
+
target=self._reader_loop, daemon=True, name="MkDB-SDK-reader"
|
|
126
|
+
)
|
|
127
|
+
reader.start()
|
|
128
|
+
|
|
129
|
+
@staticmethod
|
|
130
|
+
def _handshake_send(sock: socket.socket, msg: dict) -> None:
|
|
131
|
+
body = json.dumps(msg).encode("utf-8")
|
|
132
|
+
sock.sendall(struct.pack(">I", len(body)) + body)
|
|
133
|
+
|
|
134
|
+
@staticmethod
|
|
135
|
+
def _handshake_recv(sock: socket.socket) -> dict:
|
|
136
|
+
hdr = _recv_exact(sock, 4)
|
|
137
|
+
length = struct.unpack(">I", hdr)[0]
|
|
138
|
+
return json.loads(_recv_exact(sock, length).decode("utf-8"))
|
|
139
|
+
|
|
140
|
+
def close(self) -> None:
|
|
141
|
+
self._running = False
|
|
142
|
+
if self._sock:
|
|
143
|
+
try:
|
|
144
|
+
self._sock.close()
|
|
145
|
+
except Exception:
|
|
146
|
+
pass
|
|
147
|
+
self._sock = None
|
|
148
|
+
|
|
149
|
+
# ------------------------------------------------------------------
|
|
150
|
+
# Send / receive
|
|
151
|
+
# ------------------------------------------------------------------
|
|
152
|
+
|
|
153
|
+
def send(self, payload: dict) -> dict:
|
|
154
|
+
"""
|
|
155
|
+
Send a request and block until the matching response arrives.
|
|
156
|
+
Returns the response dict.
|
|
157
|
+
"""
|
|
158
|
+
correlation_id = str(uuid.uuid4())
|
|
159
|
+
payload["id"] = correlation_id
|
|
160
|
+
payload.setdefault("type", "request")
|
|
161
|
+
|
|
162
|
+
event = threading.Event()
|
|
163
|
+
with self._lock:
|
|
164
|
+
self._pending[correlation_id] = event
|
|
165
|
+
|
|
166
|
+
frame = _encode(payload)
|
|
167
|
+
with self._lock:
|
|
168
|
+
self._sock.sendall(frame)
|
|
169
|
+
|
|
170
|
+
event.wait(timeout=self.recv_timeout)
|
|
171
|
+
with self._lock:
|
|
172
|
+
result = self._results.pop(correlation_id, None)
|
|
173
|
+
self._pending.pop(correlation_id, None)
|
|
174
|
+
if result is None:
|
|
175
|
+
raise TimeoutError("No response received within timeout")
|
|
176
|
+
return result
|
|
177
|
+
|
|
178
|
+
def send_raw(self, payload: dict) -> None:
|
|
179
|
+
"""Fire-and-forget (used for subscribe)."""
|
|
180
|
+
frame = _encode(payload)
|
|
181
|
+
with self._lock:
|
|
182
|
+
self._sock.sendall(frame)
|
|
183
|
+
|
|
184
|
+
def register_push_handler(self, handler: Callable[[dict], None]) -> None:
|
|
185
|
+
"""Register a callback for server-push (ping, update, subscribed, disconnect)."""
|
|
186
|
+
self._push_handlers.append(handler)
|
|
187
|
+
|
|
188
|
+
# ------------------------------------------------------------------
|
|
189
|
+
# Reader loop (background thread)
|
|
190
|
+
# ------------------------------------------------------------------
|
|
191
|
+
|
|
192
|
+
def _reader_loop(self) -> None:
|
|
193
|
+
while self._running and self._sock:
|
|
194
|
+
try:
|
|
195
|
+
length_bytes = _recv_exact(self._sock, 4)
|
|
196
|
+
length = struct.unpack(">I", length_bytes)[0]
|
|
197
|
+
if length > MAX_FRAME:
|
|
198
|
+
break # protocol violation — disconnect
|
|
199
|
+
payload_bytes = _recv_exact(self._sock, length)
|
|
200
|
+
msg = _decode(payload_bytes)
|
|
201
|
+
except Exception:
|
|
202
|
+
break
|
|
203
|
+
|
|
204
|
+
msg_type = msg.get("type")
|
|
205
|
+
correlation_id = msg.get("id")
|
|
206
|
+
|
|
207
|
+
if msg_type == "response" and correlation_id:
|
|
208
|
+
with self._lock:
|
|
209
|
+
event = self._pending.get(correlation_id)
|
|
210
|
+
if event:
|
|
211
|
+
self._results[correlation_id] = msg
|
|
212
|
+
event.set()
|
|
213
|
+
elif msg_type == "ping":
|
|
214
|
+
# Respond with pong
|
|
215
|
+
try:
|
|
216
|
+
self._sock.sendall(_encode({"type": "pong"}))
|
|
217
|
+
except Exception:
|
|
218
|
+
break
|
|
219
|
+
else:
|
|
220
|
+
# Server-push: dispatch to registered handlers
|
|
221
|
+
for handler in self._push_handlers:
|
|
222
|
+
try:
|
|
223
|
+
handler(msg)
|
|
224
|
+
except Exception:
|
|
225
|
+
pass
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Delta helpers — flatten nested dicts into dot-notation keys for MkDB writes.
|
|
3
|
+
|
|
4
|
+
Example:
|
|
5
|
+
flatten({"product": {"name": "Widget", "price": 9.99}})
|
|
6
|
+
# → {"product.name": "Widget", "product.price": 9.99}
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def flatten(d: dict, prefix: str = "", sep: str = ".") -> dict:
|
|
11
|
+
"""Recursively flatten a nested dict into dot-notation keys."""
|
|
12
|
+
result = {}
|
|
13
|
+
for key, value in d.items():
|
|
14
|
+
full_key = f"{prefix}{sep}{key}" if prefix else key
|
|
15
|
+
if isinstance(value, dict):
|
|
16
|
+
result.update(flatten(value, full_key, sep))
|
|
17
|
+
else:
|
|
18
|
+
result[full_key] = value
|
|
19
|
+
return result
|