dirsql 0.3.61__tar.gz → 0.3.63__tar.gz

{dirsql-0.3.61 → dirsql-0.3.63}/Cargo.lock

@@ -500,7 +500,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.3.61"
+version = "0.3.63"
 dependencies = [
  "dirsql",
  "pyo3",

{dirsql-0.3.61 → dirsql-0.3.63}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.3.61
+Version: 0.3.63
 Requires-Dist: pytest>=8 ; extra == 'dev'
 Requires-Dist: pytest-describe>=2 ; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.23 ; extra == 'dev'

{dirsql-0.3.61 → dirsql-0.3.63}/docs/.vitepress/config.ts

@@ -2,7 +2,7 @@ import { defineConfig } from 'vitepress'
 
 export default defineConfig({
   title: 'dirsql',
-  description: 'Ephemeral SQL index over a local directory. Watches a filesystem, ingests structured files into an in-memory SQLite database, and exposes a SQL query interface.',
+  description: 'Ephemeral SQL index over a local directory. Watches a filesystem, ingests structured files into an ephemeral SQLite database, and exposes a SQL query interface.',
   base: '/dirsql/',
 
   themeConfig: {

{dirsql-0.3.61/packages/python → dirsql-0.3.63}/docs/getting-started.md

@@ -85,7 +85,7 @@ The first run downloads the package (`npx` asks for confirmation — answer
 Running at localhost:7117
 ```
 
-That one command scanned the directory, built an in-memory SQLite database
+That one command scanned the directory, built an ephemeral SQLite database
 with one row per file, and started an HTTP server. Leave it running and
 open a **second terminal** for the next step.
 

{dirsql-0.3.61 → dirsql-0.3.63}/docs/howto/persist.md

@@ -1,7 +1,7 @@
 # Keep the index across restarts
 
-By default the database lives in memory and is rebuilt from your files on
-every startup. [`persist`](../reference/config.md#dirsql-keys) keeps the
+By default the database is ephemeral: rebuilt from your files on every
+startup and discarded on exit. [`persist`](../reference/config.md#dirsql-keys) keeps the
 SQLite index on disk instead, so a restart only re-parses files that
 actually changed — the difference between seconds and milliseconds on large
 trees, and between re-running and skipping expensive

{dirsql-0.3.61 → dirsql-0.3.63}/docs/index.md

@@ -2,7 +2,7 @@
 layout: home
 hero:
   name: dirsql
-  tagline: Ephemeral SQL index over a local directory. `dirsql` watches a filesystem, ingests structured files into an in-memory SQLite database, and exposes a SQL query interface. The filesystem is always the source of truth.
+  tagline: Ephemeral SQL index over a local directory. `dirsql` watches a filesystem, ingests structured files into an ephemeral SQLite database, and exposes a SQL query interface. The filesystem is always the source of truth.
   actions:
     - theme: brand
       text: Get Started

{dirsql-0.3.61/packages/rust → dirsql-0.3.63}/docs/reference/config.md

@@ -20,7 +20,7 @@ set.
 |---|---|---|---|
 | `root` | string | config file's parent directory | Directory to index. Relative values resolve against the config file's parent. An explicit `root` passed to an SDK constructor overrides this (a warning is emitted on stderr). |
 | `ignore` | array of strings | `[]` | Glob patterns matched against root-relative paths. Matched files are skipped entirely — excluded from the initial scan and from watch events. |
-| `persist` | boolean | `false` | Keep the SQLite index on disk between runs. When `false`, the database is rebuilt in memory on every startup. |
+| `persist` | boolean | `false` | Keep the SQLite index on disk between runs. When `false`, the index is ephemeral: rebuilt from your files on every startup and discarded on exit. |
 | `persist_path` | string | `<root>/.dirsql/cache.db` | Location of the on-disk cache. Relative values resolve against the config file's parent. Ignored unless `persist = true`. |
 | `pre-query` | string | none | Server-wide command hook: the raw `POST /query` request body is passed to this command as `{args}`, and the plain-text SQL it prints is executed instead of parsing the body as `{"sql": …}`. CLI server only; the SDKs ignore it. Must be non-empty. See [Command hooks](./hooks.md#pre-query). |
 | `post-query` | string | none | Server-wide command hook: each successful `POST /query` result set is handed to this command (as a JSON array on stdin, and as `{args}` up to 96 KiB), and the JSON body it prints is returned instead of the bare row array. CLI server only; the SDKs ignore it. Must be non-empty. See [Command hooks](./hooks.md#post-query). |

{dirsql-0.3.61 → dirsql-0.3.63}/docs/reference/sdk.md

@@ -109,7 +109,7 @@ directory" error.
   When both an explicit `root` and a config root are present, the explicit
   value wins and a warning is emitted on stderr.
 - `persist` — Keep the SQLite index on disk between runs (default off:
-  in-memory, rebuilt every startup). The cache lives at
+  ephemeral, rebuilt every startup). The cache lives at
   `<root>/.dirsql/cache.db` by default; on restart, only files whose stat
   changed are re-parsed. `persist = true` in the config also enables it.
 - `persist_path` / `persistPath` — Override the cache location. An

{dirsql-0.3.61 → dirsql-0.3.63}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.3.61"
+version = "0.3.63"
 edition.workspace = true
 publish = false
 readme = "README.md"

{dirsql-0.3.61 → dirsql-0.3.63}/packages/python/docs/.vitepress/config.ts

@@ -2,7 +2,7 @@ import { defineConfig } from 'vitepress'
 
 export default defineConfig({
   title: 'dirsql',
-  description: 'Ephemeral SQL index over a local directory. Watches a filesystem, ingests structured files into an in-memory SQLite database, and exposes a SQL query interface.',
+  description: 'Ephemeral SQL index over a local directory. Watches a filesystem, ingests structured files into an ephemeral SQLite database, and exposes a SQL query interface.',
   base: '/dirsql/',
 
   themeConfig: {

{dirsql-0.3.61 → dirsql-0.3.63/packages/python}/docs/getting-started.md

@@ -85,7 +85,7 @@ The first run downloads the package (`npx` asks for confirmation — answer
 Running at localhost:7117
 ```
 
-That one command scanned the directory, built an in-memory SQLite database
+That one command scanned the directory, built an ephemeral SQLite database
 with one row per file, and started an HTTP server. Leave it running and
 open a **second terminal** for the next step.
 

{dirsql-0.3.61 → dirsql-0.3.63}/packages/python/docs/howto/persist.md

@@ -1,7 +1,7 @@
 # Keep the index across restarts
 
-By default the database lives in memory and is rebuilt from your files on
-every startup. [`persist`](../reference/config.md#dirsql-keys) keeps the
+By default the database is ephemeral: rebuilt from your files on every
+startup and discarded on exit. [`persist`](../reference/config.md#dirsql-keys) keeps the
 SQLite index on disk instead, so a restart only re-parses files that
 actually changed — the difference between seconds and milliseconds on large
 trees, and between re-running and skipping expensive

{dirsql-0.3.61/packages/rust → dirsql-0.3.63/packages/python}/docs/index.md

@@ -2,7 +2,7 @@
 layout: home
 hero:
   name: dirsql
-  tagline: Ephemeral SQL index over a local directory. `dirsql` watches a filesystem, ingests structured files into an in-memory SQLite database, and exposes a SQL query interface. The filesystem is always the source of truth.
+  tagline: Ephemeral SQL index over a local directory. `dirsql` watches a filesystem, ingests structured files into an ephemeral SQLite database, and exposes a SQL query interface. The filesystem is always the source of truth.
   actions:
     - theme: brand
       text: Get Started

{dirsql-0.3.61 → dirsql-0.3.63/packages/python}/docs/reference/config.md

@@ -20,7 +20,7 @@ set.
 |---|---|---|---|
 | `root` | string | config file's parent directory | Directory to index. Relative values resolve against the config file's parent. An explicit `root` passed to an SDK constructor overrides this (a warning is emitted on stderr). |
 | `ignore` | array of strings | `[]` | Glob patterns matched against root-relative paths. Matched files are skipped entirely — excluded from the initial scan and from watch events. |
-| `persist` | boolean | `false` | Keep the SQLite index on disk between runs. When `false`, the database is rebuilt in memory on every startup. |
+| `persist` | boolean | `false` | Keep the SQLite index on disk between runs. When `false`, the index is ephemeral: rebuilt from your files on every startup and discarded on exit. |
 | `persist_path` | string | `<root>/.dirsql/cache.db` | Location of the on-disk cache. Relative values resolve against the config file's parent. Ignored unless `persist = true`. |
 | `pre-query` | string | none | Server-wide command hook: the raw `POST /query` request body is passed to this command as `{args}`, and the plain-text SQL it prints is executed instead of parsing the body as `{"sql": …}`. CLI server only; the SDKs ignore it. Must be non-empty. See [Command hooks](./hooks.md#pre-query). |
 | `post-query` | string | none | Server-wide command hook: each successful `POST /query` result set is handed to this command (as a JSON array on stdin, and as `{args}` up to 96 KiB), and the JSON body it prints is returned instead of the bare row array. CLI server only; the SDKs ignore it. Must be non-empty. See [Command hooks](./hooks.md#post-query). |

{dirsql-0.3.61/packages/rust → dirsql-0.3.63/packages/python}/docs/reference/sdk.md

@@ -109,7 +109,7 @@ directory" error.
   When both an explicit `root` and a config root are present, the explicit
   value wins and a warning is emitted on stderr.
 - `persist` — Keep the SQLite index on disk between runs (default off:
-  in-memory, rebuilt every startup). The cache lives at
+  ephemeral, rebuilt every startup). The cache lives at
   `<root>/.dirsql/cache.db` by default; on restart, only files whose stat
   changed are re-parsed. `persist = true` in the config also enables it.
 - `persist_path` / `persistPath` — Override the cache location. An

{dirsql-0.3.61 → dirsql-0.3.63}/packages/python/e2e-attestation.json

@@ -1,6 +1,6 @@
 {
   "command": "uv run python -m pytest tests/e2e/ -x -q",
-  "ran_at": 1783171803,
+  "ran_at": 1783184817,
   "exit_code": 0,
-  "commit": "86cf2578852a1dec1149453a8af60578a16dd69c"
+  "commit": "ff790f3535072982be3fa349de4caf8c855fc406"
 }

{dirsql-0.3.61 → dirsql-0.3.63}/packages/rust/docs/getting-started.md

@@ -85,7 +85,7 @@ The first run downloads the package (`npx` asks for confirmation — answer
 Running at localhost:7117
 ```
 
-That one command scanned the directory, built an in-memory SQLite database
+That one command scanned the directory, built an ephemeral SQLite database
 with one row per file, and started an HTTP server. Leave it running and
 open a **second terminal** for the next step.
 

{dirsql-0.3.61 → dirsql-0.3.63}/packages/rust/docs/howto/persist.md

@@ -1,7 +1,7 @@
 # Keep the index across restarts
 
-By default the database lives in memory and is rebuilt from your files on
-every startup. [`persist`](../reference/config.md#dirsql-keys) keeps the
+By default the database is ephemeral: rebuilt from your files on every
+startup and discarded on exit. [`persist`](../reference/config.md#dirsql-keys) keeps the
 SQLite index on disk instead, so a restart only re-parses files that
 actually changed — the difference between seconds and milliseconds on large
 trees, and between re-running and skipping expensive

{dirsql-0.3.61/packages/python → dirsql-0.3.63/packages/rust}/docs/index.md

@@ -2,7 +2,7 @@
 layout: home
 hero:
   name: dirsql
-  tagline: Ephemeral SQL index over a local directory. `dirsql` watches a filesystem, ingests structured files into an in-memory SQLite database, and exposes a SQL query interface. The filesystem is always the source of truth.
+  tagline: Ephemeral SQL index over a local directory. `dirsql` watches a filesystem, ingests structured files into an ephemeral SQLite database, and exposes a SQL query interface. The filesystem is always the source of truth.
   actions:
     - theme: brand
       text: Get Started

{dirsql-0.3.61/packages/python → dirsql-0.3.63/packages/rust}/docs/reference/config.md

@@ -20,7 +20,7 @@ set.
 |---|---|---|---|
 | `root` | string | config file's parent directory | Directory to index. Relative values resolve against the config file's parent. An explicit `root` passed to an SDK constructor overrides this (a warning is emitted on stderr). |
 | `ignore` | array of strings | `[]` | Glob patterns matched against root-relative paths. Matched files are skipped entirely — excluded from the initial scan and from watch events. |
-| `persist` | boolean | `false` | Keep the SQLite index on disk between runs. When `false`, the database is rebuilt in memory on every startup. |
+| `persist` | boolean | `false` | Keep the SQLite index on disk between runs. When `false`, the index is ephemeral: rebuilt from your files on every startup and discarded on exit. |
 | `persist_path` | string | `<root>/.dirsql/cache.db` | Location of the on-disk cache. Relative values resolve against the config file's parent. Ignored unless `persist = true`. |
 | `pre-query` | string | none | Server-wide command hook: the raw `POST /query` request body is passed to this command as `{args}`, and the plain-text SQL it prints is executed instead of parsing the body as `{"sql": …}`. CLI server only; the SDKs ignore it. Must be non-empty. See [Command hooks](./hooks.md#pre-query). |
 | `post-query` | string | none | Server-wide command hook: each successful `POST /query` result set is handed to this command (as a JSON array on stdin, and as `{args}` up to 96 KiB), and the JSON body it prints is returned instead of the bare row array. CLI server only; the SDKs ignore it. Must be non-empty. See [Command hooks](./hooks.md#post-query). |

{dirsql-0.3.61/packages/python → dirsql-0.3.63/packages/rust}/docs/reference/sdk.md

@@ -109,7 +109,7 @@ directory" error.
   When both an explicit `root` and a config root are present, the explicit
   value wins and a warning is emitted on stderr.
 - `persist` — Keep the SQLite index on disk between runs (default off:
-  in-memory, rebuilt every startup). The cache lives at
+  ephemeral, rebuilt every startup). The cache lives at
   `<root>/.dirsql/cache.db` by default; on restart, only files whose stat
   changed are re-parsed. `persist = true` in the config also enables it.
 - `persist_path` / `persistPath` — Override the cache location. An

{dirsql-0.3.61 → dirsql-0.3.63}/packages/rust/src/config.rs

@@ -204,7 +204,8 @@ pub struct Config {
     pub ignore: Vec<String>,
     pub tables: Vec<TableConfig>,
     /// Enable persistent on-disk SQLite cache. When false (the default), the
-    /// database is rebuilt in memory on every startup.
+    /// database is rebuilt from scratch on every startup (in an anonymous
+    /// disk-backed temp database, #402).
     pub persist: bool,
     /// Optional override for the on-disk cache location. Resolved relative
     /// to the config file's parent directory when relative.

{dirsql-0.3.61 → dirsql-0.3.63}/packages/rust/src/db.rs

@@ -105,14 +105,25 @@ pub struct Db {
 }
 
 impl Db {
+    /// Open the default, ephemeral `Db`: an **anonymous disk-backed temp
+    /// database** (`Connection::open("")`, issue #402), not `:memory:`.
+    ///
+    /// SQLite creates a private temp file and deletes it immediately after
+    /// opening, so the OS reclaims it even on a crash or SIGKILL, and there
+    /// is never a name to collide on. Pages spill to disk as the index
+    /// grows — only the page cache stays resident — so memory no longer
+    /// scales with the indexed corpus. The file lands in the directory
+    /// SQLite's VFS picks (`SQLITE_TMPDIR` → `TMPDIR` → `/var/tmp` →
+    /// `/usr/tmp` → `/tmp`); export `SQLITE_TMPDIR` to steer it off a
+    /// tmpfs mount.
     pub fn new() -> Result<Self> {
-        let conn = Connection::open_in_memory()?;
+        let conn = Connection::open("")?;
         ensure_internal_rows_table(&conn)?;
         Ok(Self { conn })
     }
 
-    /// Open a `Db` backed by an on-disk SQLite file. Used by the persistent
-    /// cache path; in-memory mode is the default.
+    /// Open a `Db` backed by a named on-disk SQLite file. Used by the
+    /// persistent cache path; the anonymous temp database is the default.
     pub fn open(path: &Path) -> Result<Self> {
         let conn = Connection::open(path)?;
         ensure_internal_rows_table(&conn)?;
@@ -312,6 +323,56 @@ impl Db {
         Ok(())
     }
 
+    /// Read back the rows a given file produced for `table`, ordered by row
+    /// index. Ownership and ordering come from the `_dirsql_internal_rows`
+    /// mapping (joined on `rowid`); user columns are qualified with the table
+    /// alias so a user column named like a mapping column stays unambiguous.
+    ///
+    /// This is the watcher's diffing source (#401): `handle_upsert` /
+    /// `handle_delete` snapshot a file's previous rows here instead of holding
+    /// a corpus-wide in-memory copy. A row read back compares equal to the
+    /// normalized row that was inserted as long as the extract's value types
+    /// match the declared column affinities (SQLite coerces on insert
+    /// otherwise, e.g. `Integer(5)` into a TEXT column comes back
+    /// `Text("5")`).
+    pub fn get_rows_by_file(
+        &self,
+        table: &str,
+        file_path: &str,
+    ) -> Result<Vec<HashMap<String, Value>>> {
+        let user_columns = self.get_table_columns(table)?;
+        let mut col_list = user_columns
+            .iter()
+            .map(|c| format!("t.\"{c}\""))
+            .collect::<Vec<_>>()
+            .join(", ");
+        if col_list.is_empty() {
+            // No columns can only happen for a nonexistent table (SQLite
+            // requires ≥1 declared column); keep the SELECT list valid and
+            // let SQLite report "no such table".
+            col_list = "1".to_string();
+        }
+        let sql = format!(
+            "SELECT {col_list} FROM \"{table}\" AS t \
+             JOIN _dirsql_internal_rows AS m ON m.rowid_ref = t.rowid \
+             WHERE m.table_name = ?1 AND m.file_path = ?2 ORDER BY m.row_index"
+        );
+        let mut stmt = self.conn.prepare(&sql)?;
+        let rows = stmt.query_map(rusqlite::params![table, file_path], |row| {
+            let mut map = HashMap::new();
+            for (i, name) in user_columns.iter().enumerate() {
+                let v: rusqlite::types::Value = row.get(i)?;
+                map.insert(name.clone(), Value::from(v));
+            }
+            Ok(map)
+        })?;
+        let mut out = Vec::new();
+        for row in rows {
+            out.push(row?);
+        }
+        Ok(out)
+    }
+
     /// Delete all rows that were produced by a given file path.
     ///
     /// The user-row deletes and the matching `_dirsql_internal_rows` mapping
@@ -1206,6 +1267,95 @@ mod tests {
         assert_eq!(deleted, 0);
     }
 
+    // --- Db::new backing store (#402) ---
+
+    #[test]
+    fn new_is_disk_backed_not_memory() {
+        // An in-memory SQLite database is pinned to journal_mode=memory; a
+        // file-backed one — including the anonymous temp database `Db::new`
+        // opens — defaults to journal_mode=delete. This pins #402: the
+        // default Db must never be `:memory:`, so index pages spill to disk
+        // instead of scaling resident memory with the corpus.
+        let db = Db::new().unwrap();
+        let mode: String = db
+            .conn()
+            .query_row("PRAGMA journal_mode", [], |row| row.get(0))
+            .unwrap();
+        assert_eq!(mode, "delete");
+    }
+
+    // --- get_rows_by_file: the watcher's diffing source (#401) ---
+
+    #[test]
+    fn get_rows_by_file_returns_rows_in_row_index_order() {
+        let db = Db::new().unwrap();
+        db.create_table("CREATE TABLE t (id TEXT)").unwrap();
+        // Insert with row indices out of insertion order; the read must
+        // follow row_index, not rowid / insertion order.
+        let second = HashMap::from([("id".into(), Value::Text("second".into()))]);
+        let first = HashMap::from([("id".into(), Value::Text("first".into()))]);
+        db.insert_row("t", &second, "a.json", 1).unwrap();
+        db.insert_row("t", &first, "a.json", 0).unwrap();
+
+        let rows = db.get_rows_by_file("t", "a.json").unwrap();
+        assert_eq!(rows.len(), 2);
+        assert_eq!(rows[0]["id"], Value::Text("first".into()));
+        assert_eq!(rows[1]["id"], Value::Text("second".into()));
+    }
+
+    #[test]
+    fn get_rows_by_file_scopes_by_file_and_table() {
+        let db = Db::new().unwrap();
+        db.create_table("CREATE TABLE t (id TEXT)").unwrap();
+        db.create_table("CREATE TABLE u (id TEXT)").unwrap();
+        let t_row = HashMap::from([("id".into(), Value::Text("t-row".into()))]);
+        let u_row = HashMap::from([("id".into(), Value::Text("u-row".into()))]);
+        let other = HashMap::from([("id".into(), Value::Text("other-file".into()))]);
+        db.insert_row("t", &t_row, "a.json", 0).unwrap();
+        db.insert_row("u", &u_row, "a.json", 0).unwrap();
+        db.insert_row("t", &other, "b.json", 0).unwrap();
+
+        let rows = db.get_rows_by_file("t", "a.json").unwrap();
+        assert_eq!(rows.len(), 1);
+        assert_eq!(rows[0]["id"], Value::Text("t-row".into()));
+
+        // A file with no rows in this table reads back empty.
+        let none = db.get_rows_by_file("u", "b.json").unwrap();
+        assert!(none.is_empty());
+    }
+
+    #[test]
+    fn get_rows_by_file_round_trips_all_value_variants() {
+        // The watcher's positional diff compares a read-back row against a
+        // freshly normalized one, so every Value variant must survive
+        // insert -> select unchanged when the declared column affinity
+        // matches the value type.
+        let db = Db::new().unwrap();
+        db.create_table("CREATE TABLE t (i INTEGER, r REAL, s TEXT, b BLOB, n TEXT)")
+            .unwrap();
+        let row = HashMap::from([
+            ("i".to_string(), Value::Integer(42)),
+            ("r".to_string(), Value::Real(1.5)),
+            ("s".to_string(), Value::Text("hello".into())),
+            ("b".to_string(), Value::Blob(vec![0, 1, 2])),
+            ("n".to_string(), Value::Null),
+        ]);
+        db.insert_row("t", &row, "a.json", 0).unwrap();
+
+        let rows = db.get_rows_by_file("t", "a.json").unwrap();
+        assert_eq!(rows.len(), 1);
+        assert_eq!(rows[0], row);
+    }
+
+    #[test]
+    fn get_rows_by_file_missing_table_returns_error() {
+        // A nonexistent table has no columns, so the SELECT list falls back
+        // to `1` and SQLite reports the missing table.
+        let db = Db::new().unwrap();
+        let err = db.get_rows_by_file("ghost", "a.json").unwrap_err();
+        assert!(err.to_string().contains("no such table"), "got: {err}");
+    }
+
     // --- Error path: Db::open on an unopenable path ---
 
     #[test]