lqft-python-engine 0.2.0__tar.gz → 0.5.0__tar.gz

{lqft_python_engine-0.2.0 → lqft_python_engine-0.5.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: lqft-python-engine
-Version: 0.2.0
-Summary: LQFT Engine: Full CRUD Support & Enterprise Scaling (v4.5 Stable)
+Version: 0.5.0
+Summary: LQFT Engine: Native Disk Persistence & Cold Start Deserialization (v5.0 Stable)
 Home-page: https://github.com/ParjadM/Log-Quantum-Fractal-Tree-LQFT-
 Author: Parjad Minooei
 License: MIT

lqft_python_engine-0.5.0/lqft_engine.c

@@ -0,0 +1,490 @@
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#ifndef _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_WARNINGS 
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+/**
+ * LQFT C-Engine - V0.5.0 (Native Disk Persistence)
+ * Architect: Parjad Minooei
+ * * CHANGE LOG:
+ * - Implemented Binary Serialization (save_to_disk).
+ * - Implemented O(1) Cold Start Deserialization (load_from_disk).
+ * - DAG Pointer Reconstruction Logic.
+ */
+
+#define BIT_PARTITION 5
+#define MAX_BITS 64 
+#define MASK 0x1F 
+#define REGISTRY_SIZE 8000009 
+#define TOMBSTONE ((LQFTNode*)1)
+
+typedef struct LQFTNode {
+    void* value;
+    uint64_t key_hash;
+    struct LQFTNode* children[32]; 
+    char struct_hash[17]; 
+    uint64_t full_hash_val;
+    int ref_count;
+} LQFTNode;
+
+static LQFTNode** registry = NULL;
+static int physical_node_count = 0;
+static LQFTNode* global_root = NULL;
+
+const uint64_t FNV_OFFSET_BASIS = 14695981039346656037ULL;
+const uint64_t FNV_PRIME = 1099511628211ULL;
+
+// -------------------------------------------------------------------
+// Utilities
+// -------------------------------------------------------------------
+uint64_t fnv1a_update(uint64_t hash, const void* data, size_t len) {
+    const uint8_t* p = (const uint8_t*)data;
+    for (size_t i = 0; i < len; i++) {
+        hash ^= p[i];
+        hash *= FNV_PRIME;
+    }
+    return hash;
+}
+
+char* portable_strdup(const char* s) {
+    if (!s) return NULL;
+#ifdef _WIN32
+    return _strdup(s);
+#else
+    return strdup(s);
+#endif
+}
+
+static int init_registry() {
+    if (registry == NULL) {
+        registry = (LQFTNode**)calloc(REGISTRY_SIZE, sizeof(LQFTNode*));
+        if (registry == NULL) return 0;
+    }
+    return 1;
+}
+
+LQFTNode* create_node(void* value, uint64_t key_hash) {
+    LQFTNode* node = (LQFTNode*)malloc(sizeof(LQFTNode));
+    if (!node) return NULL;
+    node->value = value;
+    node->key_hash = key_hash;
+    node->full_hash_val = 0;
+    node->ref_count = 0;
+    for (int i = 0; i < 32; i++) node->children[i] = NULL;
+    return node;
+}
+
+// -------------------------------------------------------------------
+// Memory Management (ARC)
+// -------------------------------------------------------------------
+void decref(LQFTNode* node) {
+    if (!node) return;
+    node->ref_count--;
+    if (node->ref_count <= 0) {
+        for (int i = 0; i < 32; i++) {
+            if (node->children[i]) decref(node->children[i]);
+        }
+        uint32_t idx = node->full_hash_val % REGISTRY_SIZE;
+        uint32_t start_idx = idx;
+        while (registry[idx] != NULL) {
+            if (registry[idx] == node) {
+                registry[idx] = TOMBSTONE;
+                break;
+            }
+            idx = (idx + 1) % REGISTRY_SIZE;
+            if (idx == start_idx) break;
+        }
+        if (node->value) free(node->value);
+        free(node);
+        physical_node_count--;
+    }
+}
+
+static PyObject* method_free_all(PyObject* self, PyObject* args) {
+    if (registry != NULL) {
+        for (int i = 0; i < REGISTRY_SIZE; i++) {
+            if (registry[i] != NULL && registry[i] != TOMBSTONE) {
+                if (registry[i]->value) free(registry[i]->value);
+                free(registry[i]);
+            }
+            registry[i] = NULL;
+        }
+        free(registry);
+        registry = NULL;
+    }
+    physical_node_count = 0;
+    global_root = NULL;
+    Py_RETURN_NONE;
+}
+
+// -------------------------------------------------------------------
+// Disk Persistence (Binary Serialization)
+// -------------------------------------------------------------------
+static PyObject* method_save_to_disk(PyObject* self, PyObject* args) {
+    const char* filepath;
+    if (!PyArg_ParseTuple(args, "s", &filepath)) return NULL;
+    if (!registry) Py_RETURN_NONE;
+
+    FILE* fp = fopen(filepath, "wb");
+    if (!fp) return PyErr_SetFromErrno(PyExc_IOError);
+
+    // Header: Magic Bytes, Count, Global Root Hash
+    char magic[4] = "LQFT";
+    fwrite(magic, 1, 4, fp);
+    fwrite(&physical_node_count, sizeof(int), 1, fp);
+    uint64_t root_hash = global_root ? global_root->full_hash_val : 0;
+    fwrite(&root_hash, sizeof(uint64_t), 1, fp);
+
+    // Dump Nodes
+    for (int i = 0; i < REGISTRY_SIZE; i++) {
+        LQFTNode* node = registry[i];
+        if (node != NULL && node != TOMBSTONE) {
+            fwrite(&node->full_hash_val, sizeof(uint64_t), 1, fp);
+            fwrite(&node->key_hash, sizeof(uint64_t), 1, fp);
+            fwrite(node->struct_hash, 1, 17, fp);
+            fwrite(&node->ref_count, sizeof(int), 1, fp);
+
+            // Payload
+            int has_val = (node->value != NULL) ? 1 : 0;
+            fwrite(&has_val, sizeof(int), 1, fp);
+            if (has_val) {
+                int v_len = (int)strlen((char*)node->value);
+                fwrite(&v_len, sizeof(int), 1, fp);
+                fwrite(node->value, 1, v_len, fp);
+            }
+
+            // Children pointers (saved as full_hash_val references)
+            uint64_t child_refs[32] = {0};
+            for (int c = 0; c < 32; c++) {
+                if (node->children[c]) child_refs[c] = node->children[c]->full_hash_val;
+            }
+            fwrite(child_refs, sizeof(uint64_t), 32, fp);
+        }
+    }
+
+    fclose(fp);
+    Py_RETURN_TRUE;
+}
+
+LQFTNode* find_in_registry(uint64_t full_hash) {
+    if (full_hash == 0) return NULL;
+    uint32_t idx = full_hash % REGISTRY_SIZE;
+    uint32_t start_idx = idx;
+    while (registry[idx] != NULL) {
+        if (registry[idx] != TOMBSTONE && registry[idx]->full_hash_val == full_hash) {
+            return registry[idx];
+        }
+        idx = (idx + 1) % REGISTRY_SIZE;
+        if (idx == start_idx) break;
+    }
+    return NULL;
+}
+
+static PyObject* method_load_from_disk(PyObject* self, PyObject* args) {
+    const char* filepath;
+    if (!PyArg_ParseTuple(args, "s", &filepath)) return NULL;
+
+    FILE* fp = fopen(filepath, "rb");
+    if (!fp) return PyErr_SetFromErrno(PyExc_IOError);
+
+    char magic[5] = {0};
+    fread(magic, 1, 4, fp);
+    if (strcmp(magic, "LQFT") != 0) {
+        fclose(fp);
+        PyErr_SetString(PyExc_ValueError, "Invalid LQFT binary file format.");
+        return NULL;
+    }
+
+    // Clear current state safely
+    method_free_all(self, NULL);
+    init_registry();
+
+    int total_nodes;
+    uint64_t root_hash;
+    fread(&total_nodes, sizeof(int), 1, fp);
+    fread(&root_hash, sizeof(uint64_t), 1, fp);
+
+    // Parallel array to store child hashes for Pass 2 (Pointer Reconstruction)
+    uint64_t* all_child_refs = (uint64_t*)malloc(total_nodes * 32 * sizeof(uint64_t));
+    LQFTNode** loaded_nodes = (LQFTNode**)malloc(total_nodes * sizeof(LQFTNode*));
+
+    // PASS 1: Read physical nodes into memory
+    for (int i = 0; i < total_nodes; i++) {
+        LQFTNode* node = create_node(NULL, 0);
+        fread(&node->full_hash_val, sizeof(uint64_t), 1, fp);
+        fread(&node->key_hash, sizeof(uint64_t), 1, fp);
+        fread(node->struct_hash, 1, 17, fp);
+        fread(&node->ref_count, sizeof(int), 1, fp);
+
+        int has_val;
+        fread(&has_val, sizeof(int), 1, fp);
+        if (has_val) {
+            int v_len;
+            fread(&v_len, sizeof(int), 1, fp);
+            char* val_str = (char*)malloc(v_len + 1);
+            fread(val_str, 1, v_len, fp);
+            val_str[v_len] = '\0';
+            node->value = val_str;
+        }
+
+        fread(&all_child_refs[i * 32], sizeof(uint64_t), 32, fp);
+
+        // Insert directly into registry
+        uint32_t idx = node->full_hash_val % REGISTRY_SIZE;
+        while (registry[idx] != NULL) idx = (idx + 1) % REGISTRY_SIZE;
+        registry[idx] = node;
+        
+        loaded_nodes[i] = node;
+        physical_node_count++;
+    }
+
+    // PASS 2: Pointer Reconstruction (Relinking the DAG)
+    for (int i = 0; i < total_nodes; i++) {
+        for (int c = 0; c < 32; c++) {
+            uint64_t target_hash = all_child_refs[i * 32 + c];
+            if (target_hash != 0) {
+                loaded_nodes[i]->children[c] = find_in_registry(target_hash);
+            }
+        }
+    }
+
+    global_root = find_in_registry(root_hash);
+
+    free(all_child_refs);
+    free(loaded_nodes);
+    fclose(fp);
+
+    Py_RETURN_TRUE;
+}
+
+// -------------------------------------------------------------------
+// Merkle-DAG Core (Standard Operations)
+// -------------------------------------------------------------------
+LQFTNode* get_canonical(void* value, uint64_t key_hash, LQFTNode** children) {
+    if (!init_registry()) return NULL;
+
+    uint64_t full_hash = FNV_OFFSET_BASIS;
+    if (value != NULL) {
+        full_hash = fnv1a_update(full_hash, "leaf:", 5);
+        full_hash = fnv1a_update(full_hash, value, strlen((char*)value));
+        full_hash = fnv1a_update(full_hash, &key_hash, sizeof(uint64_t));
+    } else {
+        full_hash = fnv1a_update(full_hash, "branch:", 7);
+        if (children) {
+            for (int i = 0; i < 32; i++) {
+                if (children[i]) {
+                    full_hash = fnv1a_update(full_hash, &i, sizeof(int));
+                    full_hash = fnv1a_update(full_hash, children[i]->struct_hash, 16);
+                }
+            }
+        }
+    }
+    
+    char lookup_hash[17];
+    sprintf(lookup_hash, "%016llx", (unsigned long long)full_hash);
+    uint32_t idx = full_hash % REGISTRY_SIZE;
+    uint32_t start_idx = idx;
+    int first_tombstone = -1;
+
+    while (registry[idx] != NULL) {
+        if (registry[idx] == TOMBSTONE) {
+            if (first_tombstone == -1) first_tombstone = (int)idx;
+        } else if (registry[idx]->full_hash_val == full_hash && strcmp(registry[idx]->struct_hash, lookup_hash) == 0) {
+            if (value) free(value);
+            return registry[idx];
+        }
+        idx = (idx + 1) % REGISTRY_SIZE;
+        if (idx == start_idx) break; 
+    }
+
+    LQFTNode* new_node = create_node(value, key_hash);
+    if (!new_node) return NULL;
+    
+    if (children) {
+        for (int i = 0; i < 32; i++) {
+            new_node->children[i] = children[i];
+            if (children[i]) children[i]->ref_count++; 
+        }
+    }
+    
+    strcpy(new_node->struct_hash, lookup_hash);
+    new_node->full_hash_val = full_hash;
+    
+    uint32_t insert_idx = (first_tombstone != -1) ? (uint32_t)first_tombstone : idx;
+    registry[insert_idx] = new_node;
+    physical_node_count++;
+    
+    return new_node;
+}
+
+static PyObject* method_insert(PyObject* self, PyObject* args) {
+    unsigned long long h;
+    char* val_str;
+    if (!PyArg_ParseTuple(args, "Ks", &h, &val_str)) return NULL;
+    
+    if (!global_root) {
+        if (!init_registry()) return PyErr_NoMemory();
+        global_root = get_canonical(NULL, 0, NULL);
+        global_root->ref_count++;
+    }
+    
+    LQFTNode* old_root = global_root;
+    LQFTNode* path_nodes[20];
+    uint32_t path_segs[20];
+    int path_len = 0;
+    LQFTNode* curr = global_root;
+    int bit_depth = 0;
+    
+    while (curr != NULL && curr->value == NULL) {
+        uint32_t segment = (h >> bit_depth) & MASK;
+        path_nodes[path_len] = curr;
+        path_segs[path_len] = segment;
+        path_len++;
+        if (curr->children[segment] == NULL) { curr = NULL; break; }
+        curr = curr->children[segment];
+        bit_depth += BIT_PARTITION;
+    }
+    
+    LQFTNode* new_sub_node = NULL;
+    if (curr == NULL) { 
+        new_sub_node = get_canonical(portable_strdup(val_str), h, NULL); 
+    } else if (curr->key_hash == h) { 
+        new_sub_node = get_canonical(portable_strdup(val_str), h, curr->children); 
+    } else {
+        unsigned long long old_h = curr->key_hash;
+        char* old_val = portable_strdup((char*)curr->value);
+        int temp_depth = bit_depth;
+        while (temp_depth < 64) {
+            uint32_t s_old = (old_h >> temp_depth) & MASK;
+            uint32_t s_new = (h >> temp_depth) & MASK;
+            if (s_old != s_new) {
+                LQFTNode* c_old = get_canonical(old_val, old_h, curr->children);
+                LQFTNode* c_new = get_canonical(portable_strdup(val_str), h, NULL);
+                LQFTNode* new_children[32] = {NULL};
+                new_children[s_old] = c_old;
+                new_children[s_new] = c_new;
+                new_sub_node = get_canonical(NULL, 0, new_children);
+                break;
+            } else { 
+                path_nodes[path_len] = NULL; 
+                path_segs[path_len] = s_old; 
+                path_len++; 
+                temp_depth += BIT_PARTITION; 
+            }
+        }
+        if (new_sub_node == NULL) new_sub_node = get_canonical(portable_strdup(val_str), h, curr->children);
+    }
+    
+    for (int i = path_len - 1; i >= 0; i--) {
+        if (path_nodes[i] == NULL) {
+            LQFTNode* new_children[32] = {NULL};
+            new_children[path_segs[i]] = new_sub_node;
+            new_sub_node = get_canonical(NULL, 0, new_children);
+        } else {
+            LQFTNode* p_node = path_nodes[i];
+            uint32_t segment = path_segs[i];
+            LQFTNode* new_children[32];
+            for (int j = 0; j < 32; j++) new_children[j] = p_node->children[j];
+            new_children[segment] = new_sub_node;
+            new_sub_node = get_canonical(p_node->value, p_node->key_hash, new_children);
+        }
+    }
+    
+    global_root = new_sub_node;
+    global_root->ref_count++;
+    if (old_root) decref(old_root); 
+
+    Py_RETURN_NONE;
+}
+
+static PyObject* method_search(PyObject* self, PyObject* args) {
+    unsigned long long h;
+    if (!PyArg_ParseTuple(args, "K", &h)) return NULL;
+    if (!global_root) Py_RETURN_NONE;
+    
+    LQFTNode* curr = global_root;
+    int bit_depth = 0;
+    while (curr != NULL && curr->value == NULL) {
+        uint32_t segment = (h >> bit_depth) & MASK;
+        curr = curr->children[segment];
+        bit_depth += BIT_PARTITION;
+    }
+    
+    if (curr != NULL && curr->key_hash == h) return PyUnicode_FromString((char*)curr->value);
+    Py_RETURN_NONE;
+}
+
+static PyObject* method_delete(PyObject* self, PyObject* args) {
+    unsigned long long h;
+    if (!PyArg_ParseTuple(args, "K", &h)) return NULL;
+    if (!global_root) Py_RETURN_NONE;
+
+    LQFTNode* path_nodes[20]; 
+    uint32_t path_segs[20];
+    int path_len = 0;
+    LQFTNode* curr = global_root;
+    int bit_depth = 0;
+
+    while (curr != NULL && curr->value == NULL) {
+        uint32_t segment = (h >> bit_depth) & MASK;
+        path_nodes[path_len] = curr;
+        path_segs[path_len] = segment;
+        path_len++;
+        curr = curr->children[segment];
+        bit_depth += BIT_PARTITION;
+    }
+
+    if (curr == NULL || curr->key_hash != h) Py_RETURN_NONE;
+
+    LQFTNode* old_root = global_root;
+    LQFTNode* new_sub_node = NULL;
+
+    for (int i = path_len - 1; i >= 0; i--) {
+        LQFTNode* p_node = path_nodes[i];
+        uint32_t segment = path_segs[i];
+        LQFTNode* new_children[32];
+        int has_other_children = 0;
+
+        for (uint32_t j = 0; j < 32; j++) {
+            if (j == segment) new_children[j] = new_sub_node;
+            else {
+                new_children[j] = p_node->children[j];
+                if (new_children[j]) has_other_children = 1;
+            }
+        }
+
+        if (!has_other_children && i > 0) new_sub_node = NULL;
+        else new_sub_node = get_canonical(NULL, 0, new_children);
+    }
+
+    global_root = (new_sub_node) ? new_sub_node : get_canonical(NULL, 0, NULL);
+    if (global_root) global_root->ref_count++;
+    if (old_root) decref(old_root); 
+
+    Py_RETURN_NONE;
+}
+
+static PyObject* method_get_metrics(PyObject* self, PyObject* args) {
+    return Py_BuildValue("{s:i}", "physical_nodes", physical_node_count);
+}
+
+static PyMethodDef LQFTMethods[] = {
+    {"insert", method_insert, METH_VARARGS, "Insert key-value"},
+    {"delete", method_delete, METH_VARARGS, "Delete key"},
+    {"search", method_search, METH_VARARGS, "Search key"},
+    {"save_to_disk", method_save_to_disk, METH_VARARGS, "Serialize to .bin"},
+    {"load_from_disk", method_load_from_disk, METH_VARARGS, "Deserialize from .bin"},
+    {"get_metrics", method_get_metrics, METH_VARARGS, "Get stats"},
+    {"free_all", method_free_all, METH_VARARGS, "Total memory wipe"},
+    {NULL, NULL, 0, NULL}
+};
+
+static struct PyModuleDef lqftmodule = { PyModuleDef_HEAD_INIT, "lqft_c_engine", NULL, -1, LQFTMethods };
+PyMODINIT_FUNC PyInit_lqft_c_engine(void) { return PyModule_Create(&lqftmodule); }

{lqft_python_engine-0.2.0 → lqft_python_engine-0.5.0}/lqft_engine.py

@@ -1,8 +1,11 @@
 import hashlib
 import weakref
+import psutil
+import os
+import json
 
 # ---------------------------------------------------------
-# LEGACY PURE PYTHON LQFT (For reference/fallback/benchmarks)
+# LEGACY PURE PYTHON LQFT (For reference/fallback)
 # ---------------------------------------------------------
 class LQFTNode:
     __slots__ = ['children', 'value', 'key_hash', 'struct_hash', '__weakref__']
@@ -39,7 +42,6 @@ class LQFTNode:
         return cls._null_cache['null']
 
 class LQFT:
-    """Legacy Pure Python Iterative Implementation."""
     def __init__(self, bit_partition=5, max_bits=256):
         self.partition = bit_partition 
         self.max_bits = max_bits 
@@ -62,7 +64,7 @@ class LQFT:
                 break
             curr = curr.children[segment]
             bit_depth += self.partition
-
+            
         new_sub_node = None
         if curr is null_node:
             new_sub_node = LQFTNode.get_canonical(value, None, h)
@@ -82,7 +84,7 @@ class LQFT:
                     temp_depth += self.partition
             if new_sub_node is None:
                 new_sub_node = LQFTNode.get_canonical(value, curr.children, h)
-
+                
         for entry in reversed(path):
             if entry[0] == "split":
                 new_sub_node = LQFTNode.get_canonical(None, {entry[1]: new_sub_node}, None)
@@ -104,7 +106,7 @@ class LQFT:
         return None
 
 # ---------------------------------------------------------
-# NEW: ADAPTIVE ENTERPRISE ENGINE (MScAC Portfolio)
+# ADAPTIVE ENTERPRISE ENGINE (v0.5.0 - Disk Persistence)
 # ---------------------------------------------------------
 try:
     import lqft_c_engine
@@ -113,63 +115,92 @@ except ImportError:
     C_ENGINE_READY = False
 
 class AdaptiveLQFT:
-    """
-    A polymorphic, heuristic-driven data structure wrapper.
-    - Scale < threshold: Acts as an ultra-lightweight C-Hash (Python Dict).
-    - Scale > threshold: Automatically migrates to the Native C-Engine LQFT 
-      for Merkle-DAG deduplication and folding.
-    """
     def __init__(self, migration_threshold=50000):
         self.threshold = migration_threshold
         self.size = 0
         self.is_native = False
-        
-        # The "Mini Version": Python's highly optimized built-in dictionary
         self._light_store = {} 
+        
+        self.auto_purge_enabled = True
+        self.max_memory_mb = 1000.0 
+        self.total_ops = 0
+        self._process = psutil.Process(os.getpid())
+
+    def _validate_type(self, key, value=None):
+        if not isinstance(key, str):
+            raise TypeError(f"LQFT keys must be strings. Received: {type(key).__name__}")
+        if value is not None and not isinstance(value, str):
+            raise TypeError(f"LQFT values must be strings. Received: {type(value).__name__}")
 
     def _get_64bit_hash(self, key):
-        """Generates a 64-bit unsigned hash for the C-Engine."""
-        return int(hashlib.md5(str(key).encode()).hexdigest()[:16], 16)
+        return int(hashlib.md5(key.encode()).hexdigest()[:16], 16)
+
+    def set_auto_purge_threshold(self, threshold: float):
+        self.purge_threshold = threshold
+
+    def purge(self):
+        current_mb = self._process.memory_info().rss / (1024 * 1024)
+        if current_mb >= self.max_memory_mb:
+            print(f"\n[⚠️ CIRCUIT Breaker] Engine exceeded {self.max_memory_mb} MB limit (Currently {current_mb:.1f} MB). Auto-Purging!")
+        self.clear()
+
+    # --- Phase 1: Native Disk Persistence ---
+    def save_to_disk(self, filepath: str):
+        """Serializes the engine state to disk (O(1) Time context scaling)."""
+        if not self.is_native:
+            with open(filepath, 'w') as f:
+                json.dump(self._light_store, f)
+        else:
+            lqft_c_engine.save_to_disk(filepath)
+
+    def load_from_disk(self, filepath: str):
+        """Instantly reconstructs C-Pointers from a binary file."""
+        if not os.path.exists(filepath):
+            raise FileNotFoundError(f"Missing LQFT database file: {filepath}")
+            
+        if not self.is_native:
+            # Check if it's a C-Engine binary file (magic bytes "LQFT")
+            with open(filepath, 'rb') as f:
+                if f.read(4) == b'LQFT':
+                    self._migrate_to_native()
+                    lqft_c_engine.load_from_disk(filepath)
+                    return
+            with open(filepath, 'r') as f:
+                self._light_store = json.load(f)
+                self.size = len(self._light_store)
+        else:
+            lqft_c_engine.load_from_disk(filepath)
 
     def _migrate_to_native(self):
-        """The 'Curve Flip' mechanism: moves all data to the Heavy Engine."""
         if not C_ENGINE_READY:
-            print("[!] Warning: C-Engine missing. Staying in lightweight mode.")
-            self.threshold = float('inf') # Prevent continuous upgrade attempts
+            self.threshold = float('inf') 
             return
-            
         for key, val in self._light_store.items():
             h = self._get_64bit_hash(key)
-            lqft_c_engine.insert(h, str(val))
-            
-        # Clear the lightweight store to free up memory
+            lqft_c_engine.insert(h, val)
         self._light_store.clear()
         self.is_native = True
 
     def insert(self, key, value):
+        self._validate_type(key, value)
+        self.total_ops += 1
+        if self.auto_purge_enabled and self.total_ops % 5000 == 0:
+            current_mb = self._process.memory_info().rss / (1024 * 1024)
+            if current_mb >= self.max_memory_mb:
+                self.purge()
+
         if not self.is_native:
-            # Phase 1: Small Data Operations
-            if key not in self._light_store:
+            if key not in self._light_store: 
                 self.size += 1
             self._light_store[key] = value
-            
-            # Check for migration threshold
-            if self.size >= self.threshold:
+            if self.size >= self.threshold: 
                 self._migrate_to_native()
-        else:
-            # Phase 2: Massive Data Operations (Native C-Heap)
-            h = self._get_64bit_hash(key)
-            lqft_c_engine.insert(h, str(value))
-
-    def search(self, key):
-        if not self.is_native:
-            return self._light_store.get(key, None)
         else:
             h = self._get_64bit_hash(key)
-            return lqft_c_engine.search(h)
+            lqft_c_engine.insert(h, value)
 
     def remove(self, key):
-        """Deletes a key from either the light store or the Merkle tree."""
+        self._validate_type(key)
         if not self.is_native:
             if key in self._light_store:
                 del self._light_store[key]
@@ -179,35 +210,33 @@ class AdaptiveLQFT:
             lqft_c_engine.delete(h)
 
     def delete(self, key):
-        """Alias for remove to satisfy all testing suites."""
         self.remove(key)
 
+    def search(self, key):
+        self._validate_type(key)
+        if not self.is_native: 
+            return self._light_store.get(key, None)
+        else:
+            h = self._get_64bit_hash(key)
+            return lqft_c_engine.search(h)
+
     def clear(self):
-        """
-        Memory Reclamation: Manually trigger heap cleanup.
-        In the Adaptive model, this handles both the Python dict and the C-Registry.
-        """
         self._light_store.clear()
         self.size = 0
-        if C_ENGINE_READY:
+        if C_ENGINE_READY: 
             return lqft_c_engine.free_all()
         return 0
 
     def get_stats(self):
-        """Fetches memory metrics from the C-Engine if active."""
-        if self.is_native and C_ENGINE_READY:
+        if self.is_native and C_ENGINE_READY: 
             return lqft_c_engine.get_metrics()
         return {"physical_nodes": 0}
 
     def __del__(self):
-        """Finalizer: Reclaims unmanaged C memory when the Python object is deleted."""
-        try:
-            self.clear()
-        except:
-            pass
+        try: self.clear()
+        except: pass
 
     def status(self):
-        """Returns the current state of the engine."""
         return {
             "mode": "Native Merkle-DAG" if self.is_native else "Lightweight C-Hash",
             "items": self.size if not self.is_native else lqft_c_engine.get_metrics().get('physical_nodes', self.size),

{lqft_python_engine-0.2.0 → lqft_python_engine-0.5.0}/lqft_python_engine.egg-info/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: lqft-python-engine
-Version: 0.2.0
-Summary: LQFT Engine: Full CRUD Support & Enterprise Scaling (v4.5 Stable)
+Version: 0.5.0
+Summary: LQFT Engine: Native Disk Persistence & Cold Start Deserialization (v5.0 Stable)
 Home-page: https://github.com/ParjadM/Log-Quantum-Fractal-Tree-LQFT-
 Author: Parjad Minooei
 License: MIT

{lqft_python_engine-0.2.0 → lqft_python_engine-0.5.0}/setup.py

@@ -27,8 +27,8 @@ lqft_extension = Extension(
 
 setup(
     name="lqft-python-engine",
-    version="0.2.0", 
-    description="LQFT Engine: Full CRUD Support & Enterprise Scaling (v4.5 Stable)",
+    version="0.5.0", 
+    description="LQFT Engine: Native Disk Persistence & Cold Start Deserialization (v5.0 Stable)",
     long_description=long_description,
     long_description_content_type="text/markdown",
     author="Parjad Minooei",

lqft_python_engine-0.2.0/lqft_engine.c

@@ -1,282 +0,0 @@
-#define PY_SSIZE_T_CLEAN
-#include <Python.h>
-
-#ifndef _CRT_SECURE_NO_WARNINGS
-#define _CRT_SECURE_NO_WARNINGS 
-#endif
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stdint.h>
-
-/**
- * LQFT C-Engine - V4.5 (Full CRUD & Large Payload)
- * Architect: Parjad Minooei
- * * CHANGE LOG:
- * - Added 'delete' method with bottom-up path reconstruction.
- * - Maintains Incremental FNV-1a Hashing for large payloads.
- * - Fixed Stack-to-Heap collision for multi-MB data.
- */
-
-#define BIT_PARTITION 5
-#define MAX_BITS 64 
-#define MASK 0x1F 
-#define REGISTRY_SIZE 8000009 
-
-typedef struct LQFTNode {
-    void* value;
-    uint64_t key_hash;
-    struct LQFTNode* children[32]; 
-    char struct_hash[17]; 
-} LQFTNode;
-
-static LQFTNode** registry = NULL;
-static int physical_node_count = 0;
-static LQFTNode* global_root = NULL;
-
-const uint64_t FNV_OFFSET_BASIS = 14695981039346656037ULL;
-const uint64_t FNV_PRIME = 1099511628211ULL;
-
-uint64_t fnv1a_update(uint64_t hash, const void* data, size_t len) {
-    const uint8_t* p = (const uint8_t*)data;
-    for (size_t i = 0; i < len; i++) {
-        hash ^= p[i];
-        hash *= FNV_PRIME;
-    }
-    return hash;
-}
-
-char* portable_strdup(const char* s) {
-    if (!s) return NULL;
-#ifdef _WIN32
-    return _strdup(s);
-#else
-    return strdup(s);
-#endif
-}
-
-static int init_registry() {
-    if (registry == NULL) {
-        registry = (LQFTNode**)calloc(REGISTRY_SIZE, sizeof(LQFTNode*));
-        if (registry == NULL) return 0;
-    }
-    return 1;
-}
-
-LQFTNode* create_node(void* value, uint64_t key_hash) {
-    LQFTNode* node = (LQFTNode*)malloc(sizeof(LQFTNode));
-    if (!node) return NULL;
-    node->value = value;
-    node->key_hash = key_hash;
-    for (int i = 0; i < 32; i++) node->children[i] = NULL;
-    return node;
-}
-
-LQFTNode* get_canonical(void* value, uint64_t key_hash, LQFTNode** children) {
-    if (!init_registry()) return NULL;
-
-    uint64_t full_hash = FNV_OFFSET_BASIS;
-    if (value != NULL) {
-        full_hash = fnv1a_update(full_hash, "leaf:", 5);
-        full_hash = fnv1a_update(full_hash, value, strlen((char*)value));
-        full_hash = fnv1a_update(full_hash, &key_hash, sizeof(uint64_t));
-    } else {
-        full_hash = fnv1a_update(full_hash, "branch:", 7);
-        if (children) {
-            for (int i = 0; i < 32; i++) {
-                if (children[i]) {
-                    full_hash = fnv1a_update(full_hash, &i, sizeof(int));
-                    full_hash = fnv1a_update(full_hash, children[i]->struct_hash, 16);
-                }
-            }
-        }
-    }
-    
-    char lookup_hash[17];
-    sprintf(lookup_hash, "%016llx", (unsigned long long)full_hash);
-    uint32_t idx = full_hash % REGISTRY_SIZE;
-    uint32_t start_idx = idx;
-
-    while (registry[idx] != NULL) {
-        if (strcmp(registry[idx]->struct_hash, lookup_hash) == 0) {
-            if (value) free(value); 
-            return registry[idx];
-        }
-        idx = (idx + 1) % REGISTRY_SIZE;
-        if (idx == start_idx) break; 
-    }
-
-    LQFTNode* new_node = create_node(value, key_hash);
-    if (!new_node) return NULL;
-    if (children) {
-        for (int i = 0; i < 32; i++) new_node->children[i] = children[i];
-    }
-    strcpy(new_node->struct_hash, lookup_hash);
-    registry[idx] = new_node;
-    physical_node_count++;
-    return new_node;
-}
-
-static PyObject* method_delete(PyObject* self, PyObject* args) {
-    unsigned long long h;
-    if (!PyArg_ParseTuple(args, "K", &h)) return NULL;
-    if (!global_root) Py_RETURN_NONE;
-
-    LQFTNode* path_nodes[20]; 
-    uint32_t path_segs[20];
-    int path_len = 0;
-    LQFTNode* curr = global_root;
-    int bit_depth = 0;
-
-    // Trace path to target
-    while (curr != NULL && curr->value == NULL) {
-        uint32_t segment = (h >> bit_depth) & MASK;
-        path_nodes[path_len] = curr;
-        path_segs[path_len] = segment;
-        path_len++;
-        curr = curr->children[segment];
-        bit_depth += BIT_PARTITION;
-    }
-
-    // If not found, exit
-    if (curr == NULL || curr->key_hash != h) Py_RETURN_NONE;
-
-    // Rebuild bottom-up: start with NULL (the deleted leaf)
-    LQFTNode* new_sub_node = NULL;
-
-    for (int i = path_len - 1; i >= 0; i--) {
-        LQFTNode* p_node = path_nodes[i];
-        uint32_t segment = path_segs[i];
-        LQFTNode* new_children[32];
-        int has_other_children = 0;
-
-        for (int j = 0; j < 32; j++) {
-            if (j == segment) new_children[j] = new_sub_node;
-            else {
-                new_children[j] = p_node->children[j];
-                if (new_children[j]) has_other_children = 1;
-            }
-        }
-
-        if (!has_other_children && i > 0) {
-            new_sub_node = NULL; // Contract empty branch
-        } else {
-            new_sub_node = get_canonical(NULL, 0, new_children);
-        }
-    }
-
-    global_root = (new_sub_node) ? new_sub_node : get_canonical(NULL, 0, NULL);
-    Py_RETURN_NONE;
-}
-
-static PyObject* method_free_all(PyObject* self, PyObject* args) {
-    if (registry != NULL) {
-        for (int i = 0; i < REGISTRY_SIZE; i++) {
-            if (registry[i] != NULL) {
-                if (registry[i]->value) free(registry[i]->value);
-                free(registry[i]);
-                registry[i] = NULL;
-            }
-        }
-        free(registry);
-        registry = NULL;
-    }
-    physical_node_count = 0;
-    global_root = NULL;
-    Py_RETURN_NONE;
-}
-
-static PyObject* method_insert(PyObject* self, PyObject* args) {
-    unsigned long long h;
-    char* val_str;
-    if (!PyArg_ParseTuple(args, "Ks", &h, &val_str)) return NULL;
-    if (!global_root) {
-        if (!init_registry()) return PyErr_NoMemory();
-        global_root = get_canonical(NULL, 0, NULL);
-    }
-    LQFTNode* path_nodes[20];
-    uint32_t path_segs[20];
-    int path_len = 0;
-    LQFTNode* curr = global_root;
-    int bit_depth = 0;
-    while (curr != NULL && curr->value == NULL) {
-        uint32_t segment = (h >> bit_depth) & MASK;
-        path_nodes[path_len] = curr;
-        path_segs[path_len] = segment;
-        path_len++;
-        if (curr->children[segment] == NULL) { curr = NULL; break; }
-        curr = curr->children[segment];
-        bit_depth += BIT_PARTITION;
-    }
-    LQFTNode* new_sub_node = NULL;
-    if (curr == NULL) { new_sub_node = get_canonical(portable_strdup(val_str), h, NULL); }
-    else if (curr->key_hash == h) { new_sub_node = get_canonical(portable_strdup(val_str), h, curr->children); }
-    else {
-        unsigned long long old_h = curr->key_hash;
-        char* old_val = portable_strdup((char*)curr->value);
-        int temp_depth = bit_depth;
-        while (temp_depth < 64) {
-            uint32_t s_old = (old_h >> temp_depth) & MASK;
-            uint32_t s_new = (h >> temp_depth) & MASK;
-            if (s_old != s_new) {
-                LQFTNode* c_old = get_canonical(old_val, old_h, curr->children);
-                LQFTNode* c_new = get_canonical(portable_strdup(val_str), h, NULL);
-                LQFTNode* new_children[32] = {NULL};
-                new_children[s_old] = c_old;
-                new_children[s_new] = c_new;
-                new_sub_node = get_canonical(NULL, 0, new_children);
-                break;
-            } else { path_nodes[path_len] = NULL; path_segs[path_len] = s_old; path_len++; temp_depth += BIT_PARTITION; }
-        }
-        if (new_sub_node == NULL) new_sub_node = get_canonical(portable_strdup(val_str), h, curr->children);
-    }
-    for (int i = path_len - 1; i >= 0; i--) {
-        if (path_nodes[i] == NULL) {
-            LQFTNode* new_children[32] = {NULL};
-            new_children[path_segs[i]] = new_sub_node;
-            new_sub_node = get_canonical(NULL, 0, new_children);
-        } else {
-            LQFTNode* p_node = path_nodes[i];
-            uint32_t segment = path_segs[i];
-            LQFTNode* new_children[32];
-            for (int j = 0; j < 32; j++) new_children[j] = p_node->children[j];
-            new_children[segment] = new_sub_node;
-            new_sub_node = get_canonical(p_node->value, p_node->key_hash, new_children);
-        }
-    }
-    global_root = new_sub_node;
-    Py_RETURN_NONE;
-}
-
-static PyObject* method_search(PyObject* self, PyObject* args) {
-    unsigned long long h;
-    if (!PyArg_ParseTuple(args, "K", &h)) return NULL;
-    if (!global_root) Py_RETURN_NONE;
-    LQFTNode* curr = global_root;
-    int bit_depth = 0;
-    while (curr != NULL && curr->value == NULL) {
-        uint32_t segment = (h >> bit_depth) & MASK;
-        curr = curr->children[segment];
-        bit_depth += BIT_PARTITION;
-    }
-    if (curr != NULL && curr->key_hash == h) return PyUnicode_FromString((char*)curr->value);
-    Py_RETURN_NONE;
-}
-
-static PyObject* method_get_metrics(PyObject* self, PyObject* args) {
-    return Py_BuildValue("{s:i}", "physical_nodes", physical_node_count);
-}
-
-static PyMethodDef LQFTMethods[] = {
-    {"insert", method_insert, METH_VARARGS, "Insert key-value"},
-    {"delete", method_delete, METH_VARARGS, "Delete key"},
-    {"search", method_search, METH_VARARGS, "Search key"},
-    {"get_metrics", method_get_metrics, METH_VARARGS, "Get engine stats"},
-    {"free_all", method_free_all, METH_VARARGS, "Total memory wipe"},
-    {NULL, NULL, 0, NULL}
-};
-
-static struct PyModuleDef lqftmodule = { PyModuleDef_HEAD_INIT, "lqft_c_engine", NULL, -1, LQFTMethods };
-
-PyMODINIT_FUNC PyInit_lqft_c_engine(void) { return PyModule_Create(&lqftmodule); }