motedb-python 0.13.0__tar.gz → 0.14.0__tar.gz

{motedb_python-0.13.0 → motedb_python-0.14.0}/CHANGELOG.md

@@ -1,5 +1,52 @@
 # Changelog
 
+## [0.14.0] — 2026-10-12
+
+- **🔑 ASOF JOIN(W8, v0.14)**: 机器人帧↔状态对齐原语
+  - `t1 ASOF JOIN t2 ON key = key AND right_ts <= left_ts`(也接受
+    `left_ts >= right_ts` 等价形)— 每左行取同键下 ts 不超过它的最新右行
+  - `LEFT ASOF JOIN` 保留无匹配左行(右侧 NULL);可选键等值(缺省=全局桶)
+  - 实现: 右表按 (key → ts 升序) 建桶, 左行二分 partition_point, O(N·logM)
+  - 基建修复: **隐式别名吞 ASOF**(`fr ASOF JOIN jt` 曾解析为
+    fr{alias:"ASOF"} + Inner JOIN → 静默 0 行); normalize 操作数交换时
+    镜像不等号; 前向形式(left_ts <= right_ts)v1 明确报错
+  - 测试: test_asof_join.rs(4) — 对齐/左保留/反向形式与错误/30×40 抖动
+    时间戳 vs 测试内暴力参照逐行对拍
+- **Perf Gate 新形状**: +blob_scan(0.30x)/rrf_fusion(0.04x)/
+  traj_sim_topk(1.79x) 三个 v0.14 查询形状的比率预算(基线=全表扫描)
+- **🔑 数据集快照(W7, v0.14)**:
+  - `COMMIT DATASET 'name' [NOTE '…']` — 钉住全部用户表的当前段集
+    (段 = 不可变文件,快照即段 id 清单,近乎免费);清单持久化在
+    `_dataset_commits` 表(dataset/table/seg_id/note/committed_at)
+  - **GC 引用保护**: ColSegmentStore 增加 gc_filter(三个构造点统一安装:
+    get_or_create Vacant/Occupied 补装/open 直接构造点)—— sync_manifest
+    对被钉住的段**延迟物理删除**(留在 pending_gc 反复复查);reopen 时
+    非 active 段的清理同样改为延迟(修复:原先直接删除会吃掉被钉段,
+    kill 测试实抓)
+  - `EXPORT DATASET 'name' TO 'path'` — 把快照物化为独立新库(清单段:
+    内存句柄或按 `{id:010}.sst` 直开 Segment::open;逐活行写入新库,
+    同 schema 含 EPISODE BY;索引在目标库按需重建);导出末尾 checkpoint
+    保证即时持久化与一致首读
+  - `DROP DATASET [IF EXISTS] 'name'` — 释放钉(原生 delete_rows_batch,
+    引用集全量重建)
+  - 测试: test_datasets.rs(2) — 快照隔离(commit 前后变更不串)、
+    reopen+VACUUM 后导出仍完整(GC 保护实抓:open 路径绕过 filter 安装)、
+    DROP/IF EXISTS/未知数据集错误
+- **🔑 Episode 数据模型(W6, v0.14 第一块)**:
+  - `CREATE EPISODE TABLE … EPISODE BY (key_col, ts_col) GAP '5s'|'100ms'|
+    microseconds` — 列存在性校验, 配置持久化在 TableSchema(serde 兼容旧库)
+  - `FROM EPISODES(t)` 表函数: 按 key 分组 → ts 排序 → gap 分片, 每片段一行
+    (`<key>`, `t0`, `t1`, `n_rows`), 且每个 VECTOR 列派生 `traj_<col>` —
+    统一 64 采样线性插值重采样的展平轨迹(64×dim)。Episode 为**派生数据**
+    (分片是 key/ts/gap 的纯函数) — 无注册表需持久化, 重开一致性免费
+  - `traj_sim(traj_col, demo)` — 64×dim DTW 距离(demo 任意采样数, 内部
+    重采样到 64); W5 spike 决议的两段式架构(无物理倒排, PAA 短名单化
+    为后续优化), evaluator/positional/simple 三求值表接线
+  - 基建: 表函数参数支持裸表名 + 字符串字面量; UNION 合成支持 Vector
+    字面量(EPISODES 内联通道, 递归 CTE 路径保持标量限制)
+  - 测试: test_episodes.rs(3) — 分片正确性/形状、traj_sim vs 测试内
+    暴力 DTW 对拍、重开耐久
+
 ## [0.13.0] — 2026-10-11
 
 - **🔑 BLOB 类型(全链路)**: `payload BLOB` 列 — 第 8 个 ColumnType。

{motedb_python-0.13.0 → motedb_python-0.14.0}/Cargo.lock

@@ -921,7 +921,7 @@ dependencies = [
 
 [[package]]
 name = "motedb"
-version = "0.13.0"
+version = "0.14.0"
 dependencies = [
  "arc-swap",
  "bincode",

{motedb_python-0.13.0 → motedb_python-0.14.0}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "motedb"
-version = "0.13.0"
+version = "0.14.0"
 edition = "2021"
 rust-version = "1.87"
 description = "AI-native embedded multimodal database for embodied intelligence (robots, AR glasses, industrial arms)."

{motedb_python-0.13.0 → motedb_python-0.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: motedb-python
-Version: 0.13.0
+Version: 0.14.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research

{motedb_python-0.13.0 → motedb_python-0.14.0}/README.md

@@ -173,6 +173,50 @@ One-SQL BM25 ⊕ vector fusion (Reciprocal Rank Fusion, `k`/`rrk`/`w1..w4`
 tunable). `snippet(body, 'grasp failure', 80)` renders FTS5-style excerpts
 with `<b>` highlights.
 
+### Frame↔state alignment: ASOF JOIN (v0.14 work-in-progress)
+
+```sql
+-- For each frame, the latest joint-state at-or-before its timestamp (per session):
+SELECT f.fname, j.q FROM frames f
+ASOF JOIN joints j ON f.session = j.session AND j.ts <= f.ts;
+-- LEFT ASOF JOIN keeps frames with no prior state (NULL right side).
+```
+
+### Episode model + trajectory search (v0.14 work-in-progress)
+
+```sql
+CREATE EPISODE TABLE teleop (
+  session TEXT, ts TIMESTAMP, joint VECTOR(7), gripper FLOAT
+) EPISODE BY (session, ts) GAP '5s';
+
+-- One row per episode: bounds, row count, and a 64-sample resampled
+-- trajectory per VECTOR column (flattened 64×dim).
+SELECT session, t0, t1, n_rows FROM EPISODES(teleop);
+
+-- Trajectory similarity (DTW over the resampled series; two-stage
+-- PAA-shortlist design validated in docs/design/2026-10-traj-spike.md):
+SELECT session, t0 FROM EPISODES(teleop)
+ORDER BY traj_sim(traj_joint, [0.0, 0.1, …]) LIMIT 10;
+```
+
+Episodes are **derived data** — segmentation is a pure function of
+`(key, ts, gap)`, so there is no registry to persist or invalidate and
+reopen consistency is free.
+
+### Dataset snapshots (v0.14 work-in-progress)
+
+```sql
+COMMIT DATASET 'fixpos-v3' NOTE 'after gripper fix';
+-- … table keeps evolving (compaction GC will not touch pinned segments) …
+EXPORT DATASET 'fixpos-v3' TO '/data/fixpos-v3.mote';  -- materialized copy
+DROP DATASET 'fixpos-v3';                              -- release the pin
+```
+
+A snapshot is just the pinned segment list (segments are immutable files)
+— near-free to take, persisted in `_dataset_commits`, and compaction
+physically defers deletion of pinned files until the pin is dropped.
+EXPORT materializes the pinned state into a fresh standalone database.
+
 ### Robotics data on-ramp (new in v0.13)
 
 ```bash

{motedb_python-0.13.0 → motedb_python-0.14.0}/bindings/python/Cargo.lock

@@ -530,7 +530,7 @@ dependencies = [
 
 [[package]]
 name = "motedb"
-version = "0.13.0"
+version = "0.14.0"
 dependencies = [
  "arc-swap",
  "bincode",
@@ -558,7 +558,7 @@ dependencies = [
 
 [[package]]
 name = "motedb-python"
-version = "0.13.0"
+version = "0.14.0"
 dependencies = [
  "motedb",
  "pyo3",

{motedb_python-0.13.0 → motedb_python-0.14.0}/bindings/python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "motedb-python"
-version = "0.13.0"
+version = "0.14.0"
 edition = "2021"
 description = "Python bindings for MoteDB — AI-native embedded multimodal database."
 license = "MIT"

{motedb_python-0.13.0 → motedb_python-0.14.0}/docs/design/2026-10-embodied-roadmap.md

@@ -1,6 +1,12 @@
 # 技术方案:具身数据工作流 — v0.13「入场」与 v0.14「品类定义」
 
-> 状态:**Phase 1 完成**(W1a 文档对齐 ✅、W1b BLOB ✅ 含词法平方级修复、
+> 状态:**W8 收口完成**(ASOF JOIN 对齐原语 + Perf Gate 三新形状 +
+> v0.14 发布)→ Phase 2 全部落地。W7 数据集快照已实现(COMMIT/DROP/EXPORT + GC 引用保护,
+> 3 构造点统一 gc_filter;CHECKOUT 只读视图顺延 — EXPORT 物化覆盖训练
+> 数据管线场景)。W6 Episode 模型已实现(DDL + EPISODES(t) 表函数 + traj_sim
+> DTW 排序, 3 项集成测试含暴力 DTW 对拍)。W5 spike 决议: → 见 docs/design/2026-10-traj-spike.md:
+> SAX 倒排 NO-GO(recall 0.762<0.90),PAA+DTW 两段 GO(recall 1.000,
+> p99 26ms,开销 ~1%)—— §5-§6 产品化方案已按决议修订。Phase 1 完成(W1a 文档对齐 ✅、W1b BLOB ✅ 含词法平方级修复、
 > W2-3 MCAP 核心/CLI/Python 钩子/重放幂等 ✅、W4 RRF+bm25+snippet ✅ 含
 > 非 Table FROM 执行器修复)。顺延项:MCAP 断点续导清单表、BLOB 标量函数
 > 求值/ORDER BY 输出缺口。下一阶段:W5 轨迹索引 spike(go/no-go)。

motedb_python-0.14.0/docs/design/2026-10-traj-spike.md

@@ -0,0 +1,44 @@
+# W5 轨迹索引 Spike 备忘录 — 决议: SAX 倒排 NO-GO,PAA+DTW 两段 GO
+
+> 2026-10-11 · `examples/traj_spike.rs`(可复跑:`cargo run --release --example traj_spike`)
+> 预注册门(roadmap §5):recall@10 ≥ 0.90(structured 查询)· p99 ≤ 500ms · 索引 ≤ 8%
+
+## 实验设置
+
+- 语料 110k × 64 点:90k z-norm 随机游走 + 20k 结构化形状(10 个运动
+  族 × 2k,频率按族、相位/幅值/噪声按成员抖动 —— 机器人"动作基元"模型)
+- 查询 100:50 结构化(同族新噪声/相位)+ 50 随机游走(真值病态,只报不判)
+- 真值:并行暴力 DTW top-10(8 线程 ~11.6s)
+- 策略 A(计划方案):z-norm → PAA(8) → SAX 词(8 符号/字母表 8)→ 倒排桶;
+  查询 = Hamming≤2 邻域(1+56+1372 词)→ DTW 精排(堆满后 early-abandon)
+- 策略 B(预注册降级):PAA 欧氏 top-500 → DTW 精排(无倒排索引)
+
+## 结果(三次运行稳定)
+
+| 指标 | A SAX→DTW | B PAA→DTW | 门 |
+|---|---|---|---|
+| structured recall@10 | **0.762** | **1.000** | ≥ 0.90 |
+| p50 / p99 延迟 | 450µs / 3.4ms | 3.3ms / 25.8ms | ≤ 500ms |
+| 索引开销 | 4.7%(1.32MB) | **~1.1%**(仅 PAA 列 32B/条,无倒排) | ≤ 8% |
+| 平均候选数 | 206 | 500 | — |
+
+## 结论
+
+1. **SAX 词倒排 NO-GO**:8 符号词分辨率不足 —— 相位移动使 DTW 相似的
+   轨迹在 ≥3/8 个 PAA 段上跨越量化边界,Hamming≤2 邻域系统性漏召
+   (0.762 < 0.90)。加长词/放大邻域会指数级放大候选与实现复杂度,
+   而降级路线已满分,无迭代价值。spike 一下午击毙了复杂设计 —— 目标达成。
+2. **PAA+DTW 两段 GO**:structured recall 1.000、p99 25.8ms(门余量 19×)、
+   存储仅 PAA 列(~1%)。且架构上更契合嵌入式定位:**无独立索引结构需要
+   维护** —— 与向量检索"预计算紧凑表示 + 精确扫描 + 精排"是同一模式,
+   复用 episode close 时的异步预计算管线。
+
+## v0.14 产品化含义(修订 §5-§6)
+
+- **不建** `TRAJECTORY INDEX` 倒排;`traj_sim(trajectory, ?)` ORDER BY 实现
+  为:episode close 时预计算 PAA(8) 列(f32×8)→ 查询时欧氏 top-500
+  (可 morsel 并行,110k 单线程 3.3ms;10M 级再并行/IVF-flat)→ DTW 精排
+- 原始 z-norm 序列已在列存中(精排零额外存储);PAA 列为唯一派生数据
+- `CREATE TRAJECTORY INDEX` 语法保留为元数据标记(声明该列参与
+  traj_sim),不再是物理结构 —— 对用户心智模型不变
+- 短名单 500 有余量(B 在 250 时预计仍 ≥0.95,产品化时按 Perf Gate 调参)

{motedb_python-0.13.0 → motedb_python-0.14.0}/examples/perf_smoke.rs

@@ -58,6 +58,55 @@ fn main() {
     }
     println!("setup: {} rows", n);
 
+    // v0.14 shapes: BLOB scan, RRF fusion, traj_sim (episode DTW). A second
+    // table carries the multimodal payloads; sizes kept modest so the gate
+    // stays fast while still catching complexity-class regressions.
+    db.execute("CREATE TABLE payloads (id INT PRIMARY KEY, body TEXT, data BLOB)")
+        .unwrap();
+    for i in 0..20_000i64 {
+        db.execute(&format!(
+            "INSERT INTO payloads VALUES ({}, 'doc {} about grasp {}', X'{}')",
+            i,
+            i,
+            i % 50,
+            "ab".repeat(16)
+        ))
+        .unwrap();
+    }
+    db.execute("CREATE TEXT INDEX payloads_body ON payloads(body)")
+        .unwrap();
+    db.execute(
+        "CREATE EPISODE TABLE teleop (session TEXT, ts TIMESTAMP, joint VECTOR(2)) EPISODE BY (session, ts) GAP 1000000",
+    )
+    .unwrap();
+    for s in 0..2i64 {
+        for e in 0..200i64 {
+            let base = 1_700_000_000_000_000i64 + s * 1_000_000_000_000 + e * 2_000_000_000;
+            for k in 0..64i64 {
+                let x = ((e + k) as f64) * 0.05;
+                db.execute(&format!(
+                    "INSERT INTO teleop VALUES ('s{}', {}, [{}, 0.5])",
+                    s,
+                    base + k * 100_000,
+                    x
+                ))
+                .unwrap();
+            }
+        }
+    }
+    let demo: String = (0..64)
+        .map(|i| format!("{}", i as f64 * 0.05))
+        .collect::<Vec<_>>()
+        .join(", ");
+    let blob_hex = "ab".repeat(16);
+    let blob_scan_sql = format!("SELECT COUNT(*) FROM payloads WHERE data = X'{blob_hex}'");
+    let rrf_sql = "SELECT id FROM RRF((SELECT id FROM payloads WHERE MATCH(body, 'grasp') ORDER BY bm25(body) LIMIT 50), (SELECT id FROM payloads ORDER BY id LIMIT 50), k = 10)";
+    let traj_sql = format!(
+        "SELECT session, t0 FROM EPISODES(teleop) ORDER BY traj_sim(traj_joint, [{}]) LIMIT 10",
+        demo
+    );
+    println!("setup: {} rows (+20k payloads, 400 episodes)", n);
+
     let queries: &[(&str, &str, f64)] = &[
         ("full_scan (baseline)", "SELECT * FROM sales LIMIT 200000", f64::INFINITY),
         ("text_eq_filter", "SELECT id FROM sales WHERE region = 'US' LIMIT 100000", 8.0),
@@ -65,6 +114,9 @@ fn main() {
         ("count_sum_filter", "SELECT COUNT(*), SUM(amount) FROM sales WHERE region = 'US'", 6.0),
         ("order_topk", "SELECT id, amount FROM sales WHERE region = 'US' ORDER BY amount DESC LIMIT 10", 8.0),
         ("in_subquery", "SELECT id FROM sales WHERE customer IN (SELECT customer FROM sales WHERE region = 'US') LIMIT 100000", 14.0),
+        ("blob_scan", blob_scan_sql.as_str(), 10.0),
+        ("rrf_fusion", rrf_sql, 20.0),
+        ("traj_sim_topk", traj_sql.as_str(), 30.0),
     ];
 
     let mut baseline = Duration::ZERO;