sqlrite 0.1.14__tar.gz → 0.1.16__tar.gz

{sqlrite-0.1.14 → sqlrite-0.1.16}/Cargo.lock

@@ -3511,6 +3511,7 @@ version = "1.0.149"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
 dependencies = [
+ "indexmap 2.14.0",
  "itoa",
  "memchr",
  "serde",
@@ -3736,7 +3737,7 @@ dependencies = [
 
 [[package]]
 name = "sqlrite-desktop"
-version = "0.1.14"
+version = "0.1.16"
 dependencies = [
  "serde",
  "serde_json",
@@ -3748,7 +3749,7 @@ dependencies = [
 
 [[package]]
 name = "sqlrite-engine"
-version = "0.1.14"
+version = "0.1.16"
 dependencies = [
  "clap",
  "env_logger",
@@ -3757,13 +3758,14 @@ dependencies = [
  "prettytable-rs",
  "rustyline",
  "rustyline-derive",
+ "serde_json",
  "sqlparser",
  "thiserror 2.0.18",
 ]
 
 [[package]]
 name = "sqlrite-ffi"
-version = "0.1.14"
+version = "0.1.16"
 dependencies = [
  "cbindgen",
  "sqlrite-engine",
@@ -3771,7 +3773,7 @@ dependencies = [
 
 [[package]]
 name = "sqlrite-nodejs"
-version = "0.1.14"
+version = "0.1.16"
 dependencies = [
  "napi",
  "napi-build",
@@ -3781,7 +3783,7 @@ dependencies = [
 
 [[package]]
 name = "sqlrite-python"
-version = "0.1.14"
+version = "0.1.16"
 dependencies = [
  "pyo3",
  "sqlrite-engine",

{sqlrite-0.1.14 → sqlrite-0.1.16}/Cargo.toml

@@ -27,7 +27,7 @@ resolver = "3"
 # `package =` key so the import name stays `sqlrite` internally:
 #     sqlrite = { package = "sqlrite-engine", path = "…" }
 name = "sqlrite-engine"
-version = "0.1.14"
+version = "0.1.16"
 authors = ["Joao Henrique Machado Silva <joaoh82@gmail.com>"]
 edition = "2024"
 rust-version = "1.85"
@@ -82,6 +82,14 @@ log = "0.4"
 sqlparser = "0.61"
 thiserror = "2.0"
 prettytable-rs = "0.10"
+# Phase 7e: JSON column type. `serde_json` powers both the validation
+# step at INSERT time (parse-and-discard to confirm the text is valid
+# JSON) and the path extraction inside the json_extract / json_type
+# / json_array_length / json_object_keys SQL functions. `preserve_order`
+# keeps object keys in insertion order so json_object_keys output is
+# stable; without it, BTreeMap-backed Maps would alphabetically sort,
+# which surprises callers re-serializing the same JSON.
+serde_json = { version = "1", features = ["preserve_order"] }
 
 # CLI-only deps (feature-gated). `optional = true` + the `cli`
 # feature above means these only land in the dep graph when

{sqlrite-0.1.14 → sqlrite-0.1.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlrite
-Version: 0.1.14
+Version: 0.1.16
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License

{sqlrite-0.1.14 → sqlrite-0.1.16}/desktop/package.json

@@ -1,7 +1,7 @@
 {
   "name": "sqlrite-desktop-frontend",
   "private": true,
-  "version": "0.1.14",
+  "version": "0.1.16",
   "type": "module",
   "scripts": {
     "dev": "vite",

{sqlrite-0.1.14 → sqlrite-0.1.16}/docs/phase-7-plan.md

@@ -162,7 +162,7 @@ SELECT id, title FROM docs ORDER BY embedding <-> [0.1, ...] LIMIT 10;
 >
 > - **✅ 7d.1 — Pure HNSW algorithm** *(~700 LOC, shipped in v0.1.13).* `src/sql/hnsw.rs` standalone module: insert + search + layer assignment + beam search per layer + L2/cosine/dot distance dispatch. No SQL integration yet — vectors are passed in via a `get_vec` closure so the algorithm doesn't depend on table types. Tests verify recall@k ≥ 0.95 vs brute-force on randomly-generated vector sets; deterministic via a fixed RNG seed.
 > - **✅ 7d.2 — SQL integration** *(~500 LOC).* `CREATE INDEX … USING hnsw (col)` parser + engine, INSERT wiring (also calls `hnsw.insert()` incrementally), query optimizer hook (recognizes `ORDER BY vec_distance_l2(col, literal) LIMIT k` and probes the HNSW instead of full-scanning). HNSW lives in memory only at this point; the **CREATE INDEX SQL persists in `sqlrite_master` and reopen rebuilds the graph from current rows** — partial persistence ahead of 7d.3. DELETE/UPDATE on HNSW-indexed tables refused with helpful error pointing at 7d.3.
-> - **7d.3 — Persistence** *(~300 LOC).* Wire HNSW into the cell format: new `KIND_HNSW` cell tag, page-tree storage parallel to secondary indexes, save/reopen round-trip without rebuild. Also adds DELETE/UPDATE support since the persisted form gives us a natural rebuild trigger.
+> - **✅ 7d.3 — Persistence** *(~600 LOC).* New `KIND_HNSW` cell tag and `HnswNodeCell` encoding (varint node_id + per-layer neighbor lists). Each HNSW index gets its own page tree parallel to secondary indexes. Open path loads cells directly into `HnswIndex::from_persisted_nodes` — no algorithm runs, exact bit-for-bit reproduction. Also unblocks DELETE / UPDATE on HNSW-indexed tables: those mark the index `needs_rebuild`, save rebuilds from current rows before staging. ~2× the original 300-LOC estimate because the cell encoding + tests + rebuild path together added more than expected.
 >
 > Each 7d.x ships as its own PR + release wave. The user-facing value lands at 7d.2; 7d.3 closes the persistence loop. 7d.1 is foundational but ships a tested algorithmic primitive on its own — useful as documentation of the engine's "from scratch" theme.
 
@@ -170,14 +170,14 @@ SELECT id, title FROM docs ORDER BY embedding <-> [0.1, ...] LIMIT 10;
 
 ---
 
-### 7e — JSON column type + path queries
+### ✅ 7e — JSON column type + path queries
 
-**What.** New `JSON` data type. Store as bincoded `serde_json::Value` (or as a parsed AST — see open questions). Support a small set of extraction functions:
+**What.** New `JSON` data type. Stored as canonical UTF-8 text and validated at INSERT/UPDATE time via `serde_json::from_str`. The four path-extraction functions parse on demand:
 
 - `json_extract(col, '$.path')` — returns the value at the path, NULL if absent
-- `json_array_length(col, '$.path')` — array length, NULL for non-array
-- `json_object_keys(col, '$.path')` — TEXT array of keys, NULL for non-object
-- `json_type(col, '$.path')` — `'null'`, `'bool'`, `'number'`, `'string'`, `'array'`, `'object'`
+- `json_array_length(col, '$.path')` — array length, NULL for non-array, errors for non-array-with-path-resolved
+- `json_object_keys(col, '$.path')` — JSON-array text of keys (see scope-correction note in Q3 below; SQLite's set-returning shape requires features we don't have)
+- `json_type(col, '$.path')` — `'null'` / `'true'` / `'false'` / `'integer'` / `'real'` / `'text'` / `'array'` / `'object'` (matches SQLite JSON1 conventions)
 
 **Why this matters for AI-era specifically.** LLM tool-call outputs are JSON. RAG citation arrays are JSON. Agent scratchpads are JSON. Storing them as TEXT and re-parsing on every query is wasteful.
 
@@ -378,6 +378,10 @@ Q1–Q10 were resolved by the project owner on 2026-04-26. Each question keeps i
 ### Q3. JSON storage format
 
 > **Decided: bincoded `serde_json::Value`** for the MVP. JSON indexing remains a future phase.
+>
+> **Scope correction (2026-04-28, during 7e implementation):** Q3's "bincoded `Value`" answer was settled before remembering that bincode was removed from the engine in Phase 3c (cell-based encoding replaced it). Rather than re-add bincode for one column type, **7e ships JSON-as-canonical-text** — same as SQLite's JSON1 extension. INSERT/UPDATE call `serde_json::from_str` to validate; the four `json_*` functions re-parse on demand. Trade-off: ~2× storage vs. binary, plus per-call parse overhead — both acceptable for MVP and consistent with SQLite's choice. JSONB-style binary indexing remains a future-phase optimization, but doesn't block 7e.
+>
+> One additional 7e divergence from the original plan: `json_object_keys` is supposed to be a *table-valued function* (one row per key, like SQLite's). We don't yet support set-returning functions in the executor, so 7e returns the keys as a JSON-array text instead. Caller can iterate via `json_array_length` + `json_extract` indexing. Documented in `docs/supported-sql.md` so users see the divergence up front.
 
 - **bincoded `serde_json::Value`:** one-line implementation, fast read/write, opaque on disk.
 - **Parsed AST as cell-encoded structure:** more code, but lets us index into JSON without a full deserialize.

{sqlrite-0.1.14 → sqlrite-0.1.16}/docs/roadmap.md

@@ -473,8 +473,8 @@ Approved sub-phases (Q1–Q10 resolved):
 - **✅ 7a — `VECTOR(N)` column type** *(v0.1.10)* — dense fixed-dimension f32 storage via the existing cell encoding; format bumped to v4. Bracket-array literal syntax `[0.1, 0.2, …]` (Q7).
 - **✅ 7b — Distance functions** *(v0.1.11)* — `vec_distance_l2/cosine/dot`, plus the ORDER BY-expressions parser change so KNN queries work end-to-end. Operators (`<->` `<=>` `<#>`) deferred to **7b.1** — sqlparser doesn't parse them natively, contradicting Q6's "tiny parser change" assumption.
 - **✅ 7c — Brute-force KNN executor optimization** — bounded `BinaryHeap` of size k for `ORDER BY <expr> LIMIT k`. ~1.8× faster than full-sort at N=10k for cheap keys; bigger gains on expensive keys like `vec_distance_l2`.
-- **7d — HNSW ANN index** — split into 7d.1 (✅ algorithm), 7d.2 (✅ SQL integration), 7d.3 (persistence). `CREATE INDEX … USING hnsw (col)`; fixed defaults `M=16, ef_construction=200, ef_search=50` (Q2).
-- **7e — JSON column type + path queries** — `JSON` data type stored as bincoded `serde_json::Value` (Q3); `json_extract` / `json_array_length` / `json_object_keys` / `json_type`.
+- **✅ 7d — HNSW ANN index** — three PRs: 7d.1 (algorithm w/ recall@10 ≥ 0.95), 7d.2 (SQL integration + query optimizer), 7d.3 (persistence + DELETE/UPDATE rebuild). `CREATE INDEX … USING hnsw (col)`; fixed defaults `M=16, ef_construction=200, ef_search=50` (Q2). New `KIND_HNSW` cell tag.
+- **✅ 7e — JSON column type + path queries** — `JSON` data type stored as canonical text (validated via `serde_json::from_str` at INSERT/UPDATE time; SQLite-JSON1-style — Q3 scope correction since bincode was removed in Phase 3c). Functions: `json_extract` / `json_type` / `json_array_length` / `json_object_keys`. Path subset supports `$`, `.key`, `[N]`, chained. `json_object_keys` returns a JSON-array text rather than a table-valued result (no set-returning functions in the executor yet).
 - **7f — ~~Full-text search with BM25~~** — **deferred to Phase 8** (Q1).
 - **7g — `ask()` API across the product surface** — natural-language → SQL via Anthropic API (Q4), Anthropic-first then OpenAI + Ollama follow-ups. Foundational 7g.1 introduces a new `sqlrite-ask` crate (Q10 — separate crate, not a feature flag). Thin per-product adapters in 7g.2-7g.8 cover REPL, desktop, Python, Node.js, Go, WASM (JS-callback shape per Q9), and the MCP `ask` tool.
 - **7h — MCP server adapter** — new `sqlrite-mcp` binary, hand-rolled JSON-RPC + tool framework (Q5).

{sqlrite-0.1.14 → sqlrite-0.1.16}/docs/supported-sql.md

@@ -35,6 +35,7 @@ CREATE TABLE <name> (<col> <type> [column_constraint]* [, ...]);
 | `REAL`, `FLOAT`, `DOUBLE`, `DECIMAL` | Real (f64) | Double-precision; `DECIMAL(p,s)` precision/scale parsed and ignored |
 | `BOOLEAN` | Boolean | Stored compactly in the null bitmap's sibling bits; accepts `TRUE` / `FALSE` |
 | `VECTOR(N)` | Vector (Vec\<f32\>, fixed dim N) | **Phase 7a.** Dense f32 array of fixed dimension. `N` is required and must be ≥ 1. Inserted as bracket-array literals `[0.1, 0.2, ...]`. Dimension is enforced at INSERT/UPDATE; mismatched-length values are rejected. Distance functions and ANN indexing land in 7b–7d. |
+| `JSON`, `JSONB` | Text (canonical JSON) | **Phase 7e.** JSON document stored as canonical UTF-8 text — same as SQLite's JSON1 extension (Q3 scope correction since bincode was removed in Phase 3c). INSERT/UPDATE values are validated via `serde_json::from_str`; malformed JSON is rejected with a typed error and no row is written. `JSONB` is accepted as an alias for `JSON` (PostgreSQL convention; both store as text in our case). Path-style read access via the `json_extract` / `json_type` / `json_array_length` / `json_object_keys` functions below. |
 
 ### Column constraints
 
@@ -203,6 +204,10 @@ Same set accepted by `INSERT` (see [Value literals accepted](#value-literals-acc
 | `vec_distance_l2(a, b)` | Real (f64) | Euclidean distance √Σ(aᵢ−bᵢ)². Smaller is closer. *(Phase 7b)* |
 | `vec_distance_cosine(a, b)` | Real (f64) | Cosine distance `1 − (a·b) / (‖a‖·‖b‖)`. Errors on zero-magnitude vectors (cosine is undefined). Smaller is closer; identical vectors return 0.0, orthogonal vectors return 1.0. *(Phase 7b)* |
 | `vec_distance_dot(a, b)` | Real (f64) | Negated dot product `−(a·b)`. Negation makes "smaller is closer" consistent with the others. For unit-norm vectors equals `vec_distance_cosine(a, b) - 1`. *(Phase 7b)* |
+| `json_extract(json, path)` | Depends on the resolved node | Walks `path` over `json` and returns the resolved value coerced to the closest SQL type — JSON strings → `TEXT`, numbers → `INTEGER` / `REAL`, booleans → `BOOLEAN`, `null` → `NULL`, and composites (`object` / `array`) → their canonical JSON-text serialization. Path defaults to `$` when only one argument is supplied. A path that doesn't resolve returns `NULL`. *(Phase 7e)* |
+| `json_type(json[, path])` | Text | One of `'object'`, `'array'`, `'string'`, `'integer'`, `'real'`, `'true'`, `'false'`, `'null'`. Path defaults to `$`. *(Phase 7e)* |
+| `json_array_length(json[, path])` | Integer | Number of elements in the JSON array at `path`. Errors if the resolved node is not an array. Path defaults to `$`. *(Phase 7e)* |
+| `json_object_keys(json[, path])` | Text (JSON-array string) | Returns the object's keys as a JSON-array text in insertion order — e.g. `'["a","b","c"]'`. Path defaults to `$`. **Diverges from SQLite**, which exposes keys as a *table-valued* function (one row per key). SQLRite has no set-returning functions yet, so we return the keys as a JSON array and let callers parse if needed. *(Phase 7e)* |
 
 All three vector-distance functions take exactly two arguments, both of which must be vectors of the same dimension. Either argument can be a column reference (`embedding`), a bracket-array literal (`[0.1, 0.2, 0.3]`), or any sub-expression that evaluates to a vector. Mismatched dimensions error with `vector dimensions don't match (lhs=N, rhs=M)`.
 
@@ -216,6 +221,34 @@ LIMIT 10;
 
 > **Operator forms (`<->` `<=>` `<#>`) are not supported yet.** They're the de facto pgvector convention but blocked on a sqlparser limitation — will land as a Phase 7b.1 follow-up. Use the function-call form for now.
 
+#### JSON path syntax
+
+The `json_*` functions accept a string path argument with a small subset of JSONPath:
+
+| Token | Meaning |
+|---|---|
+| `$` | Root of the document (default if path is omitted). |
+| `.key` | Object member access. Bare keys only — no quoted-string variant yet. |
+| `[N]` | Array index (0-based). Negative indices are not supported. |
+
+Tokens chain naturally: `$.user.tags[0]`, `$[2].name`, `$.matrix[1][0]`. A malformed path (unbalanced brackets, missing `$`) errors at runtime with a typed message; a well-formed path that simply doesn't resolve returns `NULL`.
+
+```sql
+CREATE TABLE events (id INTEGER PRIMARY KEY, payload JSON);
+
+INSERT INTO events (payload) VALUES
+  ('{"user": {"name": "alice", "tags": ["admin", "ops"]}, "score": 42}'),
+  ('{"user": {"name": "bob", "tags": []}, "score": 7}');
+
+SELECT id,
+       json_extract(payload, '$.user.name') AS name,
+       json_extract(payload, '$.user.tags[0]') AS first_tag,
+       json_array_length(payload, '$.user.tags') AS tag_count,
+       json_type(payload, '$.score') AS score_type
+  FROM events
+ WHERE json_extract(payload, '$.user.name') = 'alice';
+```
+
 ### Type coercion in arithmetic
 
 - **Integer-only ops stay integer.** `1 + 2` → `3` (Integer).

{sqlrite-0.1.14 → sqlrite-0.1.16}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "sqlrite"
-version = "0.1.14"
+version = "0.1.16"
 description = "Python bindings for SQLRite — a small, embeddable SQLite clone written in Rust."
 authors = [{ name = "Joao Henrique Machado Silva", email = "joaoh82@gmail.com" }]
 license = { text = "MIT" }

{sqlrite-0.1.14 → sqlrite-0.1.16}/sdk/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sqlrite-python"
-version = "0.1.14"
+version = "0.1.16"
 authors = ["Joao Henrique Machado Silva <joaoh82@gmail.com>"]
 edition = "2024"
 rust-version = "1.85"

{sqlrite-0.1.14 → sqlrite-0.1.16}/src/sql/db/table.rs

@@ -26,6 +26,16 @@ pub enum DataType {
     /// declared dimension; every value stored in the column must have
     /// exactly that many elements.
     Vector(usize),
+    /// Phase 7e — JSON column. Stored as canonical UTF-8 text (matches
+    /// SQLite's JSON1 extension), validated at INSERT time. The
+    /// `json_extract` family of functions parses on demand and returns
+    /// either a primitive `Value` (Integer / Real / Text / Bool / Null)
+    /// or a Text value carrying the JSON-encoded sub-object/array.
+    /// Q3 originally specified `bincoded serde_json::Value`, but bincode
+    /// was removed from the engine in Phase 3c — see the scope-correction
+    /// note in `docs/phase-7-plan.md` for the rationale on switching to
+    /// text storage.
+    Json,
     None,
     Invalid,
 }
@@ -44,6 +54,7 @@ impl DataType {
             "text" => DataType::Text,
             "real" => DataType::Real,
             "bool" => DataType::Bool,
+            "json" => DataType::Json,
             "none" => DataType::None,
             other if other.starts_with("vector(") && other.ends_with(')') => {
                 // Strip the `vector(` prefix and trailing `)`, parse what's
@@ -77,6 +88,7 @@ impl DataType {
             DataType::Real => "Real".to_string(),
             DataType::Bool => "Bool".to_string(),
             DataType::Vector(dim) => format!("vector({dim})"),
+            DataType::Json => "Json".to_string(),
             DataType::None => "None".to_string(),
             DataType::Invalid => "Invalid".to_string(),
         }
@@ -91,6 +103,7 @@ impl fmt::Display for DataType {
             DataType::Real => f.write_str("Real"),
             DataType::Bool => f.write_str("Boolean"),
             DataType::Vector(dim) => write!(f, "Vector({dim})"),
+            DataType::Json => f.write_str("Json"),
             DataType::None => f.write_str("None"),
             DataType::Invalid => f.write_str("Invalid"),
         }
@@ -143,6 +156,11 @@ pub struct HnswIndexEntry {
     pub column_name: String,
     /// The graph itself.
     pub index: HnswIndex,
+    /// Phase 7d.3 — true iff a DELETE or UPDATE-on-vector-col has
+    /// invalidated the graph since the last rebuild. INSERT maintains
+    /// the graph incrementally and leaves this false. The next save
+    /// rebuilds dirty indexes from current rows before serializing.
+    pub needs_rebuild: bool,
 }
 
 impl Table {
@@ -178,6 +196,12 @@ impl Table {
                 // itself doesn't carry the dim — every stored Vec<f32>
                 // already has it via .len().
                 DataType::Vector(_dim) => Row::Vector(BTreeMap::new()),
+                // Phase 7e — JSON columns reuse Text storage (with
+                // INSERT-time validation that the bytes parse as JSON).
+                // No new Row variant; json_extract / json_type / etc.
+                // re-parse from text on demand. See `docs/phase-7-plan.md`
+                // Q3's scope-correction note for the storage choice.
+                DataType::Json => Row::Text(BTreeMap::new()),
                 DataType::Invalid | DataType::None => Row::None,
             };
             table_rows
@@ -535,7 +559,16 @@ impl Table {
                 (Row::Real(m), Value::Integer(v), _) => {
                     m.insert(rowid, *v as f32);
                 }
-                (Row::Text(m), Value::Text(v), _) => {
+                (Row::Text(m), Value::Text(v), dt) => {
+                    // Phase 7e — UPDATE on a JSON column also validates
+                    // the new text is well-formed JSON, mirroring INSERT.
+                    if matches!(dt, DataType::Json) {
+                        if let Err(e) = serde_json::from_str::<serde_json::Value>(v) {
+                            return Err(SQLRiteError::General(format!(
+                                "Type mismatch: expected JSON for column '{column}', got '{v}': {e}"
+                            )));
+                        }
+                    }
                     m.insert(rowid, v.clone());
                 }
                 (Row::Bool(m), Value::Bool(v), _) => {
@@ -650,6 +683,14 @@ impl Table {
                     }
                     Value::Vector(parsed_vec)
                 }
+                DataType::Json => {
+                    // JSON values stored as Text. UNIQUE on a JSON column
+                    // compares the canonical text representation
+                    // verbatim — `{"a": 1}` and `{"a":1}` are distinct.
+                    // Document this if anyone actually requests UNIQUE
+                    // JSON; for MVP, treat-as-text is fine.
+                    Value::Text(val.clone())
+                }
                 DataType::None | DataType::Invalid => {
                     return Err(SQLRiteError::Internal(format!(
                         "column '{name}' has an unsupported datatype"
@@ -779,6 +820,19 @@ impl Table {
                         Some(Value::Integer(parsed as i64))
                     }
                     Row::Text(tree) => {
+                        // Phase 7e — JSON columns also reach here (they
+                        // share Row::Text storage with TEXT columns).
+                        // Validate the value parses as JSON before
+                        // storing; otherwise we'd happily write
+                        // `not-json-at-all` and only fail when
+                        // json_extract tried to parse it later.
+                        if matches!(self.columns[i].datatype, DataType::Json) && val != "Null" {
+                            if let Err(e) = serde_json::from_str::<serde_json::Value>(&val) {
+                                return Err(SQLRiteError::General(format!(
+                                    "Type mismatch: expected JSON for column '{key}', got '{val}': {e}"
+                                )));
+                            }
+                        }
                         tree.insert(next_rowid, val.to_string());
                         // "Null" sentinel stays out of the index — it isn't a
                         // real user value.