dirsql 0.3.59__tar.gz → 0.3.61__tar.gz

{dirsql-0.3.59 → dirsql-0.3.61}/Cargo.lock

@@ -500,7 +500,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.3.59"
+version = "0.3.61"
 dependencies = [
  "dirsql",
  "pyo3",

{dirsql-0.3.59 → dirsql-0.3.61}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.3.59
+Version: 0.3.61
 Requires-Dist: pytest>=8 ; extra == 'dev'
 Requires-Dist: pytest-describe>=2 ; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.23 ; extra == 'dev'

dirsql-0.3.61/docs/.vitepress/config.ts

@@ -0,0 +1,86 @@
+import { defineConfig } from 'vitepress'
+
+export default defineConfig({
+  title: 'dirsql',
+  description: 'Ephemeral SQL index over a local directory. Watches a filesystem, ingests structured files into an in-memory SQLite database, and exposes a SQL query interface.',
+  base: '/dirsql/',
+
+  themeConfig: {
+    search: {
+      provider: 'local'
+    },
+
+    // The nav mirrors the four Diataxis groups exactly (#353/#387). Type is
+    // the only organizational axis -- no product-area tabs (the old `CLI`
+    // tab is gone).
+    nav: [
+      { text: 'Tutorial', link: '/getting-started' },
+      { text: 'How-to Guides', link: '/howto/define-tables' },
+      { text: 'Reference', link: '/reference/cli' },
+      { text: 'Explanation', link: '/explanation' },
+      { text: 'GitHub', link: 'https://github.com/thekevinscott/dirsql' }
+    ],
+
+    // A single global sidebar shown on every page, mirroring the four
+    // Diataxis groups. There is intentionally no path-scoped (e.g.
+    // `/howto/`) key: a path-scoped sidebar swaps the whole tree out, which
+    // deletes the other sections when you enter one (see #301). Keep one
+    // sidebar.
+    sidebar: {
+      '/': [
+        {
+          text: 'Tutorial',
+          items: [
+            { text: 'Your first dirsql database', link: '/getting-started' }
+          ]
+        },
+        {
+          text: 'How-to Guides',
+          items: [
+            { text: 'Define tables for your files', link: '/howto/define-tables' },
+            { text: 'Derive columns from file paths', link: '/howto/columns-from-paths' },
+            { text: 'Extract rows from file contents', link: '/howto/extract-from-contents' },
+            { text: 'Search documents by meaning', link: '/howto/search-by-meaning' },
+            { text: "Skip files you don't want indexed", link: '/howto/skip-files' },
+            { text: 'Load a SQLite extension', link: '/howto/load-extension' },
+            { text: 'Keep the index across restarts', link: '/howto/persist' },
+            { text: 'React to file changes', link: '/howto/react-to-changes' },
+            { text: 'Embed dirsql in your application', link: '/howto/embed' }
+          ]
+        },
+        {
+          text: 'Reference',
+          items: [
+            { text: 'CLI', link: '/reference/cli' },
+            { text: 'Configuration File', link: '/reference/config' },
+            { text: 'Command Hooks', link: '/reference/hooks' },
+            { text: 'Virtual Columns & Glob Captures', link: '/reference/columns' },
+            { text: 'HTTP API', link: '/reference/http-api' },
+            { text: 'SDK', link: '/reference/sdk' },
+            { text: 'Migrations', link: '/migrations' }
+          ]
+        },
+        {
+          text: 'Explanation',
+          items: [
+            { text: 'How dirsql thinks', link: '/explanation' }
+          ]
+        }
+      ]
+    },
+
+    outline: {
+      level: [2, 3],
+      label: 'On this page'
+    },
+
+    socialLinks: [
+      { icon: 'github', link: 'https://github.com/thekevinscott/dirsql' }
+    ],
+
+    footer: {
+      message: 'Released under the MIT License.',
+      copyright: 'Copyright 2024-present'
+    }
+  }
+})

dirsql-0.3.61/docs/AGENTS.md

@@ -0,0 +1,98 @@
+# Documentation Development
+
+Instructions for agents working on `dirsql` documentation.
+
+## Stack
+
+The docs site uses [VitePress](https://vitepress.dev/). Source files are in `docs/` at the project root.
+
+## Running locally
+
+```bash
+cd docs
+pnpm install
+pnpm dev
+```
+
+This starts a local dev server (default: `http://localhost:5173/dirsql/`). The site hot-reloads on file changes.
+
+## Building
+
+```bash
+cd docs
+pnpm build
+```
+
+The build must succeed before pushing. VitePress will fail on broken links, missing assets, and syntax errors in markdown.
+
+## Testing changes
+
+Before pushing any docs changes:
+
+1. Run `pnpm build` in `docs/` and confirm it exits cleanly
+2. Spot-check the built output with `pnpm preview`
+3. Verify sidebar navigation, code blocks, and internal links render correctly
+
+## Structure
+
+The docs follow the [Diataxis](https://diataxis.fr/) framework. **Type is
+the only organizational axis** -- there are no product-area sections (no
+"CLI" section; #353). The nav and the sidebar mirror the four types exactly.
+
+The **primary reader is the CLI user**: someone with a directory of files,
+one command (`uvx` / `npx dirsql`), and a `.dirsql.toml`. The SDKs are the
+secondary audience and appear **only in Reference**, plus the single
+"Embed `dirsql` in your application" how-to.
+
+Target tree (the spec for #353; existing pages are *quarried* into it, not
+migrated -- a page survives only if a slot wants its content):
+
+- **Tutorial** (`getting-started.md`) -- one lesson: *Your first dirsql
+  database*. The reader performs every step and sees output at each one;
+  success is author-guaranteed (toy dataset, no branching).
+- **How-to Guides** (`howto/`) -- goal-named recipes: define tables for your files;
+  derive columns from file paths; extract rows from file contents
+  (`on-file`); search documents by meaning; skip files; load a SQLite
+  extension; keep the index across restarts; react to file changes; embed
+  `dirsql` in an application.
+- **Reference** (`reference/`) -- CLI flags and defaults; the complete
+  `.dirsql.toml` schema; the command hook contract (placeholders, stdout
+  protocol, exit codes, timeouts); virtual columns and glob captures; the
+  HTTP API; the SDK page (`reference/sdk.md`, one page with
+  Python/TypeScript/Rust code-groups -- the sole SDK home); plus the
+  Migrations include (`migrations.md`).
+- **Explanation** (`explanation.md`) -- one page: how `dirsql` thinks (the
+  filesystem is the source of truth; the database is a derived, ephemeral,
+  read-only view; reconcile and diffing). Its canonical home is the root
+  `ARCHITECTURE.md`;
+  #374 surfaces it via an include page, the same mechanism
+  `docs/migrations.md` uses for `MIGRATIONS.md`. Edit the root file, never
+  a rendered include.
+
+Working rules:
+
+- **Facts live once, in Reference.** Tutorials and how-tos link to reference
+  material; they never re-list constructor parameters or duplicate API tables.
+- **A how-to opens with a 1-2 line goal/motivation statement.** Deep
+  rationale (tradeoffs, alternatives considered, theory) moves to Explanation
+  only when it is substantial enough to stand alone and is reused across
+  pages. Do not manufacture stub pages for a paragraph of "why".
+- **Tutorial vs how-to:** a tutorial is a lesson along a path the author
+  guarantees; a how-to serves a competent reader pursuing their own goal.
+- **One global sidebar.** Never add a path-scoped sidebar key -- it swaps
+  out the whole tree and hides every other section while inside one (#301;
+  see the comment above `sidebar` in `config.ts`).
+
+## Conventions
+
+- **Lead with the use case.** Open each feature description with *why* a
+  reader would reach for it before *how* it works. Don't frame a feature
+  by what an adjacent feature can't do.
+  *Don't:* "Persistence avoids the thing the default mode can't do..."
+  *Do:* "Persistence keeps the SQLite index on disk between runs so large
+  directories don't re-scan on every startup."
+- Wrap `dirsql` in backticks in all prose text
+- Use VitePress [code group](https://vitepress.dev/guide/markdown#code-groups) syntax (`::: code-group`) for multi-language examples with `Python`, `Rust`, and `TypeScript` tabs
+- Internal links use relative paths (e.g., `./howto/define-tables.md`)
+- The VitePress config is at `docs/.vitepress/config.ts`
+- The site is deployed under the `/dirsql/` base path

dirsql-0.3.61/docs/explanation.md

@@ -0,0 +1,5 @@
+---
+title: How dirsql thinks
+---
+
+<!--@include: ../ARCHITECTURE.md-->

dirsql-0.3.61/docs/getting-started.md

@@ -0,0 +1,257 @@
+# Your first dirsql database
+
+In this tutorial you will turn a directory of three tiny markdown files into
+a SQL database you can query over HTTP — without writing any code. You will:
+
+1. Create the directory and files.
+2. Start `dirsql` with zero configuration and query it with `curl`.
+3. Define your own table in a `.dirsql.toml` and query the new shape.
+
+It takes about five minutes.
+
+**You need:** a terminal with `curl` and [`jq`](https://jqlang.org/), and
+Node ≥ 20.11 (for `npx`). Every `npx dirsql` step below also has a `uvx`
+tab that behaves identically, if you prefer Python tooling
+([`uv`](https://docs.astral.sh/uv/)).
+
+## 1. Create three files
+
+Make a working directory with two subfolders — one per note author:
+
+```bash
+mkdir -p my-notes/notes/alice my-notes/notes/bob
+cd my-notes
+```
+
+Create the three notes by pasting each block exactly as shown:
+
+```bash
+cat > notes/alice/welcome.md <<'EOF'
+# Welcome
+
+Start here. This folder is about to become a database.
+EOF
+```
+
+```bash
+cat > notes/alice/ideas.md <<'EOF'
+# Ideas
+
+- query files with SQL
+- watch for changes
+EOF
+```
+
+```bash
+cat > notes/bob/reading-list.md <<'EOF'
+# Reading list
+
+- The SQLite file format
+EOF
+```
+
+Check that all three files are in place:
+
+```bash
+find notes -type f | sort
+```
+
+```
+notes/alice/ideas.md
+notes/alice/welcome.md
+notes/bob/reading-list.md
+```
+
+## 2. Start the server
+
+From inside `my-notes`, start `dirsql`:
+
+::: code-group
+
+```bash [npm]
+npx dirsql
+```
+
+```bash [PyPI]
+uvx dirsql
+```
+
+:::
+
+The first run downloads the package (`npx` asks for confirmation — answer
+`y`; `uvx` prints download progress), then the server starts:
+
+```
+Running at localhost:7117
+```
+
+That one command scanned the directory, built an in-memory SQLite database
+with one row per file, and started an HTTP server. Leave it running and
+open a **second terminal** for the next step.
+
+## 3. Query your files
+
+You gave `dirsql` no configuration, so it serves a single default table
+named `files` ([zero-config mode](./reference/cli.md#zero-config-mode)).
+Ask it how many rows it has:
+
+```bash
+curl -s http://localhost:7117/query \
+  -H 'content-type: application/json' \
+  -d '{"sql":"SELECT COUNT(*) AS files FROM files"}'
+```
+
+```
+[{"files":3}]
+```
+
+Three files, three rows. The response is always a JSON array of row
+objects ([HTTP API](./reference/http-api.md)) — from here on we pipe it
+through `jq` to pretty-print. Now select some columns:
+
+```bash
+curl -s http://localhost:7117/query \
+  -H 'content-type: application/json' \
+  -d '{"sql":"SELECT _path, _size FROM files ORDER BY _path"}' \
+  | jq
+```
+
+```json
+[
+  {
+    "_path": "notes/alice/ideas.md",
+    "_size": 52
+  },
+  {
+    "_path": "notes/alice/welcome.md",
+    "_size": 66
+  },
+  {
+    "_path": "notes/bob/reading-list.md",
+    "_size": 41
+  }
+]
+```
+
+`_path` and `_size` are two of the built-in file columns `dirsql` collects
+for every file — see [virtual columns](./reference/columns.md#virtual-columns)
+for the full list. (The `_size` values are byte counts; they match the
+output above because you pasted the files exactly.)
+
+You have a working SQL database over your files. Next, teach it the
+structure your folders already encode.
+
+## 4. Define a table
+
+Look at the paths again: `notes/alice/ideas.md`, `notes/bob/reading-list.md`
+— the author's name is a directory segment. A config file can capture it as
+a real column.
+
+In your second terminal, still inside `my-notes`, create a `.dirsql.toml`:
+
+```bash
+cat > .dirsql.toml <<'EOF'
+[[table]]
+ddl  = "CREATE TABLE notes (author TEXT, _basename TEXT, _size INTEGER)"
+glob = "notes/{author}/*.md"
+EOF
+```
+
+Two keys define the table:
+
+- `glob` selects which files feed the table, and `{author}` is a
+  [glob capture](./reference/columns.md#glob-captures): whatever directory
+  name matches that segment becomes the row's `author` value.
+- `ddl` is ordinary `CREATE TABLE` SQL naming the columns you want to keep.
+
+## 5. Restart and query the new shape
+
+Config is read at startup, so go back to the **first terminal**, stop the
+server with `Ctrl-C`, and start it again:
+
+::: code-group
+
+```bash [npm]
+npx dirsql
+```
+
+```bash [PyPI]
+uvx dirsql
+```
+
+:::
+
+```
+Running at localhost:7117
+```
+
+This time `dirsql` found your `.dirsql.toml` and served the `notes` table
+you defined instead of the default `files` table. Query it from the second
+terminal:
+
+```bash
+curl -s http://localhost:7117/query \
+  -H 'content-type: application/json' \
+  -d '{"sql":"SELECT author, _basename, _size FROM notes ORDER BY author, _basename"}' \
+  | jq
+```
+
+```json
+[
+  {
+    "_basename": "ideas.md",
+    "_size": 52,
+    "author": "alice"
+  },
+  {
+    "_basename": "welcome.md",
+    "_size": 66,
+    "author": "alice"
+  },
+  {
+    "_basename": "reading-list.md",
+    "_size": 41,
+    "author": "bob"
+  }
+]
+```
+
+Every row now carries an `author` column extracted from its path — no
+extraction code, just a glob. And it is a real SQL column, so you can
+aggregate on it:
+
+```bash
+curl -s http://localhost:7117/query \
+  -H 'content-type: application/json' \
+  -d '{"sql":"SELECT author, COUNT(*) AS notes FROM notes GROUP BY author"}' \
+  | jq
+```
+
+```json
+[
+  {
+    "author": "alice",
+    "notes": 2
+  },
+  {
+    "author": "bob",
+    "notes": 1
+  }
+]
+```
+
+That's the whole loop: files in a directory, a declarative table on top,
+SQL over HTTP.
+
+## Where to go next
+
+- [Configuration file](./reference/config.md) — the complete `.dirsql.toml`
+  reference: more tables, ignore patterns, persistence, hooks.
+- [CLI](./reference/cli.md) — flags like `--port` and `--config`, plus
+  `dirsql init`.
+- [HTTP API](./reference/http-api.md) — `POST /query` in full, plus
+  `GET /events`, a live stream of row changes as files change.
+- [SDK](./reference/sdk.md) — embed `dirsql` in a Python, Rust, or
+  TypeScript program instead of running the server.
+- Why is the database rebuilt from your files on every startup? See
+  [how `dirsql` thinks](./explanation.md).

dirsql-0.3.61/docs/howto/columns-from-paths.md

@@ -0,0 +1,63 @@
+# Derive columns from file paths
+
+Directory layouts often encode real data — an author, a year, a thread ID —
+as path segments. A `{name}` capture in a table's glob turns such a segment
+into a queryable column, no extraction code required.
+
+## 1. Name the segment in the glob
+
+Suppose photos are filed by year and month:
+
+```
+photos/2024/05/beach.jpg
+photos/2024/11/hike.jpg
+photos/2025/01/snow.jpg
+```
+
+Capture both directory levels in `.dirsql.toml`:
+
+```toml
+[[table]]
+ddl  = "CREATE TABLE photos (year TEXT, month TEXT, _basename TEXT)"
+glob = "photos/{year}/{month}/*.jpg"
+```
+
+A capture only populates a column when the DDL declares one with the same
+name — here `year` and `month`. The capture rules (valid names, matching
+within one path segment) are in
+[glob captures](../reference/columns.md#glob-captures).
+
+## 2. Query the captured columns
+
+Start the server (`npx dirsql` / `uvx dirsql`) and query:
+
+```bash
+curl -s http://localhost:7117/query \
+  -H 'content-type: application/json' \
+  -d '{"sql":"SELECT year, month, _basename FROM photos ORDER BY year, month"}'
+```
+
+```json
+[{"_basename":"beach.jpg","month":"05","year":"2024"},{"_basename":"hike.jpg","month":"11","year":"2024"},{"_basename":"snow.jpg","month":"01","year":"2025"}]
+```
+
+Captures are real SQL columns, so aggregation works:
+
+```bash
+curl -s http://localhost:7117/query \
+  -H 'content-type: application/json' \
+  -d '{"sql":"SELECT year, COUNT(*) AS photos FROM photos GROUP BY year"}'
+```
+
+```json
+[{"photos":2,"year":"2024"},{"photos":1,"year":"2025"}]
+```
+
+## Going further
+
+- Captures combine freely with [virtual columns](../reference/columns.md#virtual-columns)
+  (`_basename` above) — both are filesystem facts merged onto every row.
+- The [tutorial](../getting-started.md) walks the same idea with an
+  `{author}` capture, starting from zero.
+- When the value you need lives inside the file rather than in its path,
+  see [Extract rows from file contents](./extract-from-contents.md).

dirsql-0.3.61/docs/howto/define-tables.md

@@ -0,0 +1,70 @@
+# Define tables for your files
+
+Map a glob of files to a named SQL table so you query exactly the files you
+care about, with exactly the columns you care about — instead of the
+catch-all `files` table that [zero-config mode](../reference/cli.md#zero-config-mode)
+serves.
+
+## 1. Create a config next to your files
+
+Suppose your blog posts live under `posts/`, one markdown file each. In the
+directory you want to index, create a `.dirsql.toml` with one
+[`[[table]]`](../reference/config.md#table) entry:
+
+```toml
+[[table]]
+ddl  = "CREATE TABLE posts (_path TEXT, _size INTEGER, _mtime INTEGER)"
+glob = "posts/**/*.md"
+```
+
+- `glob` selects the files: every `.md` under `posts/`, at any depth,
+  relative to the directory containing the config.
+- `ddl` is a plain SQLite `CREATE TABLE` naming the columns you want. Here
+  all three are [virtual columns](../reference/columns.md#virtual-columns) —
+  filesystem facts `dirsql` computes for every file. Facts are opt-in by
+  DDL: only the ones you declare become columns.
+
+## 2. Start the server and query
+
+::: code-group
+
+```bash [npm]
+npx dirsql
+```
+
+```bash [PyPI]
+uvx dirsql
+```
+
+:::
+
+Each matched file is one row:
+
+```bash
+curl -s http://localhost:7117/query \
+  -H 'content-type: application/json' \
+  -d '{"sql":"SELECT _path, _size FROM posts ORDER BY _path"}'
+```
+
+```json
+[{"_path":"posts/2024/hello.md","_size":21},{"_path":"posts/2025/again.md","_size":55}]
+```
+
+Files that don't match the glob (a `README.txt` next to `posts/`, say) are
+simply not in the table. Once a config file exists, it fully replaces the
+zero-config default — only the tables you define are served.
+
+## Multiple tables
+
+Add one `[[table]]` entry per table. When a file matches several globs, the
+first matching table wins — see [`[[table]]`](../reference/config.md#table)
+for that and the remaining keys (`strict`, `on-file`).
+
+## Going further
+
+- Your directory layout encodes data (authors, dates, IDs)? Capture path
+  segments as columns — [Derive columns from file paths](./columns-from-paths.md).
+- Need columns from *inside* the files? A plain table never reads file
+  contents — [Extract rows from file contents](./extract-from-contents.md).
+- Why one row per file, rebuilt from disk? See
+  [how `dirsql` thinks](../explanation.md).