dirsql 0.2.9__tar.gz → 0.3.1__tar.gz

{dirsql-0.2.9 → dirsql-0.3.1}/Cargo.lock

@@ -73,6 +73,18 @@ version = "1.0.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
 
+[[package]]
+name = "arrayref"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
+
+[[package]]
+name = "arrayvec"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
+
 [[package]]
 name = "assert_cmd"
 version = "2.2.1"
@@ -176,6 +188,20 @@ version = "2.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
 
+[[package]]
+name = "blake3"
+version = "1.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e"
+dependencies = [
+ "arrayref",
+ "arrayvec",
+ "cc",
+ "cfg-if",
+ "constant_time_eq",
+ "cpufeatures",
+]
+
 [[package]]
 name = "bstr"
 version = "1.12.1"
@@ -300,6 +326,12 @@ version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
 
+[[package]]
+name = "constant_time_eq"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
+
 [[package]]
 name = "convert_case"
 version = "0.11.0"
@@ -325,6 +357,15 @@ version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
+[[package]]
+name = "cpufeatures"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "criterion"
 version = "0.5.1"
@@ -392,27 +433,6 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
 
-[[package]]
-name = "csv"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938"
-dependencies = [
- "csv-core",
- "itoa",
- "ryu",
- "serde_core",
-]
-
-[[package]]
-name = "csv-core"
-version = "0.1.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "ctor"
 version = "0.10.0"
@@ -441,9 +461,9 @@ version = "0.2.7"
 dependencies = [
  "assert_cmd",
  "axum",
+ "blake3",
  "clap",
  "criterion",
- "csv",
  "eventsource-client",
  "futures",
  "futures-channel",
@@ -458,7 +478,6 @@ dependencies = [
  "rusqlite",
  "serde",
  "serde_json",
- "serde_yaml",
  "tempfile",
  "thiserror",
  "tokio",
@@ -480,7 +499,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.2.9"
+version = "0.3.1"
 dependencies = [
  "dirsql",
  "pyo3",
@@ -2188,19 +2207,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "serde_yaml"
-version = "0.9.34+deprecated"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
-dependencies = [
- "indexmap",
- "itoa",
- "ryu",
- "serde",
- "unsafe-libyaml",
-]
-
 [[package]]
 name = "shlex"
 version = "1.3.0"
@@ -2591,12 +2597,6 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
 
-[[package]]
-name = "unsafe-libyaml"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
-
 [[package]]
 name = "untrusted"
 version = "0.9.0"

{dirsql-0.2.9 → dirsql-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.2.9
+Version: 0.3.1
 Requires-Dist: pytest>=8 ; extra == 'dev'
 Requires-Dist: pytest-describe>=2 ; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.23 ; extra == 'dev'

{dirsql-0.2.9 → dirsql-0.3.1}/docs/.vitepress/config.ts

@@ -32,6 +32,7 @@ export default defineConfig({
           { text: 'Defining Tables', link: '/guide/tables' },
           { text: 'Querying', link: '/guide/querying' },
           { text: 'File Watching', link: '/guide/watching' },
+          { text: 'Persistence', link: '/guide/persistence' },
           { text: 'Async API', link: '/guide/async' },
           { text: 'Command-Line Interface', link: '/guide/cli' },
           { text: 'Collaboration with CRDTs', link: '/guide/crdt' }

dirsql-0.3.1/docs/guide/config.md

@@ -0,0 +1,193 @@
+---
+canonical: https://thekevinscott.github.io/dirsql/guide/config
+---
+
+# Configuration File
+
+> Online: <https://thekevinscott.github.io/dirsql/guide/config>
+
+`dirsql` can be configured with a `.dirsql.toml` file. Tables defined this
+way produce **one row per matched file**. Each row's columns come from
+filesystem facts:
+
+- **Glob path captures** — named `{placeholder}` segments in the glob.
+- **Stat virtuals** — reserved `_`-prefixed columns for path-derived and
+  stat-derived metadata.
+
+Content interpretation (parsing JSON, CSV, frontmatter, etc.) is **not**
+configured in `.dirsql.toml`. If you need columns derived from file
+contents, register a programmatic [`Table`](./tables.md) whose `extract`
+function does the parsing in your host language.
+
+## Basic Example
+
+```toml
+[dirsql]
+ignore = ["node_modules/**", ".git/**"]
+
+[[table]]
+ddl  = "CREATE TABLE posts (_path TEXT, _basename TEXT, _size INTEGER, _mtime INTEGER)"
+glob = "posts/*.md"
+```
+
+Each `posts/*.md` file produces one row. The DDL declares which stat
+virtuals are surfaced as SQL columns.
+
+## Loading a Config File
+
+Pass the config file path to the `DirSQL` constructor:
+
+::: code-group
+
+```python [Python]
+from dirsql import DirSQL
+
+db = DirSQL(config="./my-project/.dirsql.toml")
+await db.ready()
+```
+
+```rust [Rust]
+use dirsql::DirSQL;
+
+let db = DirSQL::builder()
+    .config("./my-project/.dirsql.toml")
+    .build()?;
+```
+
+```typescript [TypeScript]
+import { DirSQL } from "dirsql";
+
+// String argument is interpreted as a config file path.
+const db = new DirSQL("./my-project/.dirsql.toml");
+await db.ready;
+```
+
+:::
+
+By default, the root directory scanned is the config file's parent
+directory. Override it by passing `root` explicitly (the explicit value
+wins and a warning is emitted) or by declaring `[dirsql].root` in the
+config file itself.
+
+## Root Directory
+
+By default, the config file's parent directory is the scan root. To index
+a different location, declare `[dirsql].root` (relative paths are resolved
+relative to the config file's parent):
+
+```toml
+[dirsql]
+root = "../data"
+ignore = ["node_modules/**"]
+```
+
+## Stat Virtuals
+
+Every config-defined table can expose any of these reserved columns. Add
+the ones you want to your DDL; the rest are silently dropped.
+
+| Column | Type    | Source |
+|--------|---------|--------|
+| `_path`     | TEXT    | The file's path relative to the scan root. |
+| `_basename` | TEXT    | The filename including extension. |
+| `_dir`      | TEXT    | The parent directory path (relative to root). |
+| `_ext`      | TEXT    | The file extension, lowercased, no leading dot. |
+| `_size`     | INTEGER | Size in bytes. |
+| `_mtime`    | INTEGER | Last-modified time, unix seconds. |
+| `_ctime`    | INTEGER | Created/changed time, unix seconds. |
+
+Example query:
+
+```sql
+SELECT _basename, _size
+FROM posts
+WHERE _mtime > strftime('%s', '2024-01-01')
+ORDER BY _mtime DESC;
+```
+
+## Path Captures
+
+Use `{name}` in glob patterns to extract path segments as columns. Add a
+matching column name to the DDL and the capture is auto-populated:
+
+```toml
+[[table]]
+ddl  = "CREATE TABLE comments (thread_id TEXT, _basename TEXT, _mtime INTEGER)"
+glob = "_comments/{thread_id}/*.jsonl"
+```
+
+A file at `_comments/abc123/2024-05-05.jsonl` produces a row with
+`thread_id = "abc123"`, `_basename = "2024-05-05.jsonl"`, and `_mtime` set
+to the file's modification time.
+
+## Ignore Patterns
+
+The `ignore` list skips files and directories entirely (not even scanned):
+
+```toml
+[dirsql]
+ignore = ["node_modules/**", ".git/**", "*.pyc", "__pycache__/**"]
+```
+
+The top-level `.dirsql/` directory is always excluded, whether you list it
+or not — it is a reserved namespace for `dirsql`'s own metadata (see
+[Persistence](./persistence.md)).
+
+## Persistence
+
+Set `persist = true` to keep the SQLite database on disk between runs
+instead of rebuilding from scratch on every startup:
+
+```toml
+[dirsql]
+persist = true
+# persist_path = ".dirsql/cache.db"   # optional; this is the default
+```
+
+See [Persistence](./persistence.md) for the full reconcile algorithm,
+storage layout, and limitations.
+
+## Strict Mode
+
+By default, auto-injected virtuals that aren't in the DDL are silently
+dropped, and undeclared user-extract keys are dropped. Enable strict mode
+to error when an extract emits keys not declared in the DDL:
+
+```toml
+[[table]]
+ddl  = "CREATE TABLE comments (thread_id TEXT)"
+glob = "_comments/{thread_id}/*.jsonl"
+strict = true
+```
+
+Strict mode does **not** apply to auto-injected stat virtuals — those are
+always filtered to the DDL's declared columns regardless. Strict mode
+applies only to keys produced by an extract callback (relevant for
+programmatic [tables](./tables.md)).
+
+## Full Example
+
+```toml
+[dirsql]
+ignore = ["node_modules/**", ".git/**", "dist/**"]
+
+[[table]]
+ddl  = "CREATE TABLE comments (thread_id TEXT, _basename TEXT, _mtime INTEGER)"
+glob = "_comments/{thread_id}/*.jsonl"
+
+[[table]]
+ddl  = "CREATE TABLE documents (_path TEXT, _basename TEXT, _size INTEGER)"
+glob = "**/index.md"
+
+[[table]]
+ddl  = "CREATE TABLE logs (_path TEXT, _size INTEGER, _mtime INTEGER)"
+glob = "logs/*.csv"
+```
+
+## When you need parsed content
+
+`.dirsql.toml` does not parse file contents. For columns derived from the
+*inside* of files (frontmatter keys, JSON values, CSV cells, etc.),
+register a programmatic [`Table`](./tables.md) instead, and parse the
+bytes in your host language. Glob captures and stat virtuals are still
+auto-injected into rows produced by your extract.

dirsql-0.3.1/docs/guide/persistence.md

@@ -0,0 +1,177 @@
+# Persistence
+
+By default `dirsql` keeps its SQLite database in memory and rebuilds it from scratch every time the process starts. For large directories this can take seconds to minutes -- nearly all of which is spent re-parsing files that haven't changed since the previous run.
+
+Persistence stores the SQLite database on disk so that subsequent startups only re-parse the files that have actually changed.
+
+::: tip Same answers, faster startup
+The rows returned by `query()` after a persistent startup are equivalent to those produced by a from-scratch rebuild. Persistence is a startup-time optimization, not a correctness compromise. The reconcile algorithm is the same one `git status` uses to decide which files have changed since the last index write.
+:::
+
+## Quick start
+
+::: code-group
+
+```toml [.dirsql.toml]
+[dirsql]
+persist = true
+```
+
+```python [Python]
+from dirsql import DirSQL
+
+db = DirSQL("./my-project", tables=[...], persist=True)
+await db.ready()
+```
+
+```rust [Rust]
+use dirsql::DirSQL;
+
+let db = DirSQL::builder()
+    .root("./my-project")
+    .tables(vec![/* ... */])
+    .persist(true)
+    .build()?;
+```
+
+```typescript [TypeScript]
+import { DirSQL } from "dirsql";
+
+const db = new DirSQL({ root: "./my-project", tables: [/* ... */], persist: true });
+await db.ready;
+```
+
+:::
+
+That's it. The first run writes the database to `./my-project/.dirsql/cache.db`. Every subsequent startup uses the cache.
+
+## Configuration
+
+| Option | Type | Default | Meaning |
+|---|---|---|---|
+| `persist` | boolean | `false` | Enable persistent on-disk storage. |
+| `persist_path` (Python, Rust) / `persistPath` (TypeScript) | string | `<root>/.dirsql/cache.db` | Override the database file path. Ignored when `persist` is `false`. |
+
+The default location keeps the cache alongside the data it indexes, which means it follows the project around (clone, copy, move) without extra setup. Override `persist_path` if you want the cache somewhere else -- a CI cache directory, a tmpfs mount, an XDG cache dir, etc.
+
+::: code-group
+
+```toml [.dirsql.toml]
+[dirsql]
+persist = true
+persist_path = "/var/cache/dirsql/myproject.db"
+```
+
+```python [Python]
+db = DirSQL(
+    "./my-project",
+    tables=[...],
+    persist=True,
+    persist_path="/var/cache/dirsql/myproject.db",
+)
+```
+
+```rust [Rust]
+let db = DirSQL::builder()
+    .root("./my-project")
+    .tables(vec![/* ... */])
+    .persist(true)
+    .persist_path("/var/cache/dirsql/myproject.db")
+    .build()?;
+```
+
+```typescript [TypeScript]
+const db = new DirSQL({
+  root: "./my-project",
+  tables: [/* ... */],
+  persist: true,
+  persistPath: "/var/cache/dirsql/myproject.db",
+});
+```
+
+:::
+
+## The `.dirsql/` directory
+
+`dirsql` reserves the top-level `.dirsql/` directory inside every scanned root. It is **unconditionally excluded from the directory walk**, whether persistence is enabled or not. This means:
+
+- The default cache path `<root>/.dirsql/cache.db` cannot accidentally be ingested as a data file.
+- You can place additional `dirsql`-related files in `.dirsql/` (e.g. a project-local config snapshot) without them being parsed.
+- You should not put your own data files in `.dirsql/` -- they will be silently ignored.
+
+If you persist into `.dirsql/`, add it to your `.gitignore`:
+
+```
+.dirsql/
+```
+
+The cache file should never be committed -- it is reproducible from the source tree and frequently large.
+
+## How the startup reconcile works
+
+When a persistent cache exists, `dirsql` does not blindly trust it. On startup it:
+
+1. **Checks compatibility metadata.** If the cached `dirsql` version, schema version, glob configuration, parser versions, or canonical root path differs from the current build, the cache is wiped and rebuilt from scratch.
+2. **Walks the tree and stats every matching file.** This is metadata-only -- no file contents are read.
+3. **For each file, compares the live `(size, mtime, ctime, inode, dev)` tuple against the cached row:**
+   - **Trust the cache** when every field matches *and* the file's mtime is older than the cache's snapshot time (outside the racy window).
+   - **Hash-confirm** when the tuple matches but the file's mtime falls inside the racy window. `dirsql` reads and hashes the file; if the hash matches the cached hash, the cache is trusted.
+   - **Re-parse** when any field of the tuple differs.
+4. **Deletes** rows for files that were in the cache but are no longer on disk.
+5. **Inserts** rows for files that are on disk but were not in the cache.
+
+This is the same algorithm `git status` uses to decide which files have changed since the last index write. The "racy window" handling is what closes the gap when a file is modified within the same filesystem-timestamp resolution as the cache write.
+
+## When `dirsql` does a full rebuild
+
+Any of the following will cause the cache to be discarded and rebuilt from scratch on the next startup:
+
+- The `dirsql` library was upgraded between runs.
+- The glob configuration changed (a new table, a removed table, a modified glob, a changed `ignore` list).
+- A built-in parser version changed (this generally only happens on `dirsql` upgrades).
+- The cache was written for a different root directory than the one currently configured.
+- The internal schema of the cache changed (i.e. you upgraded `dirsql` across a schema version bump).
+
+Full rebuilds take exactly as long as a non-persistent startup -- there is no penalty for them, only a missed optimization.
+
+## Limitations
+
+### Network filesystems
+
+NFS, SMB/CIFS, and similar network filesystems cache file attributes on the client and can return stale `stat` results. Persistent mode is **not supported** on network filesystems and may produce stale rows. Use in-memory mode (the default) if your `root` lives on a network mount.
+
+### The mtime-preservation edge case
+
+Racy-stat detection misses changes only when **all** of the following are true:
+
+- A file's contents are modified.
+- The file's size after modification is identical to its size before.
+- The file's `mtime` is externally reset to a value older than the cache's snapshot time (e.g. via `touch -r` or a backup-restore tool that preserves mtime).
+
+If you cannot tolerate this edge case, disable persistence (`persist = false`). This is the same trade-off `git` makes with `core.trustctime` / `core.checkStat`.
+
+### Single writer
+
+Only one `dirsql` process should write to a given cache file at a time. Multiple read-only processes can query the same file safely once the writer finishes the initial reconcile. Coordinated multi-writer access is not supported in v0.3.0.
+
+## Inspecting the cache
+
+The persistent database is a normal SQLite file. You can open it with any SQLite client:
+
+```bash
+sqlite3 .dirsql/cache.db
+```
+
+```sql
+.tables
+-- comments  documents  metrics  _dirsql_files  _dirsql_meta
+
+SELECT * FROM _dirsql_meta;
+-- schema_version    | 1
+-- dirsql_version    | 0.3.0
+-- glob_config_hash  | <hex>
+-- parser_versions   | {"json":"1","jsonl":"1","csv":"1",...}
+-- root_canonical    | /home/alice/my-project
+```
+
+The `_dirsql_files` and `_dirsql_meta` tables are managed by `dirsql`. Do not modify them by hand -- on the next startup, `dirsql` will detect the inconsistency and rebuild from scratch.

{dirsql-0.2.9 → dirsql-0.3.1}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.2.9"
+version = "0.3.1"
 edition.workspace = true
 publish = false
 readme = "README.md"

{dirsql-0.2.9 → dirsql-0.3.1}/packages/python/docs/.vitepress/config.ts

@@ -32,6 +32,7 @@ export default defineConfig({
           { text: 'Defining Tables', link: '/guide/tables' },
           { text: 'Querying', link: '/guide/querying' },
           { text: 'File Watching', link: '/guide/watching' },
+          { text: 'Persistence', link: '/guide/persistence' },
           { text: 'Async API', link: '/guide/async' },
           { text: 'Command-Line Interface', link: '/guide/cli' },
           { text: 'Collaboration with CRDTs', link: '/guide/crdt' }