dirsql 0.3.119__tar.gz → 0.3.121__tar.gz

{dirsql-0.3.119 → dirsql-0.3.121}/Cargo.lock

@@ -500,7 +500,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.3.119"
+version = "0.3.121"
 dependencies = [
  "dirsql",
  "pyo3",

{dirsql-0.3.119 → dirsql-0.3.121}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.3.119
+Version: 0.3.121
 Summary: Ephemeral SQL index over a local directory
 Keywords: sql,filesystem,directory,sqlite,index
 Author: Kevin Scott

{dirsql-0.3.119 → dirsql-0.3.121}/docs/.vitepress/config.ts

@@ -41,6 +41,7 @@ export default defineConfig({
             { text: 'Define tables for your files', link: '/howto/define-tables' },
             { text: 'Derive columns from file paths', link: '/howto/columns-from-paths' },
             { text: 'Extract rows from file contents', link: '/howto/extract-from-contents' },
+            { text: 'Parse your files into columns', link: '/howto/parse-files-into-columns' },
             { text: 'Search documents by meaning', link: '/howto/search-by-meaning' },
             { text: "Skip files you don't want indexed", link: '/howto/skip-files' },
             { text: 'Load a SQLite extension', link: '/howto/load-extension' },

dirsql-0.3.121/docs/getting-started.md

@@ -0,0 +1,248 @@
+# Your first dirsql database
+
+In this tutorial you will turn a directory of three tiny markdown files into
+a SQL database — with a single command, and without writing any code. You
+will:
+
+1. Create the directory and files.
+2. Query them straight away with zero configuration.
+3. Declare your own table to name and reuse a shape, and query it.
+
+It takes about five minutes.
+
+`dirsql` only ever reads your files — it never writes, moves, or changes
+them — so it is safe to point at a real directory of your own once you are
+done here. See [Read-only by design](./explanation#read-only-by-design).
+
+**You need:** a terminal with [`jq`](https://jqlang.org/), and Node ≥ 20.11
+(for `npx`). Every `npx dirsql` step below also has a `uvx` tab that behaves
+identically, if you prefer Python tooling
+([`uv`](https://docs.astral.sh/uv/)).
+
+## 1. Create three files
+
+Paste this whole block into your terminal. It makes a working directory
+with a subfolder per note author and writes three tiny markdown notes:
+
+```bash
+mkdir -p my-notes/notes/alice my-notes/notes/bob
+cd my-notes
+cat > notes/alice/welcome.md <<'EOF'
+# Welcome
+
+Start here. This folder is about to become a database.
+EOF
+cat > notes/alice/ideas.md <<'EOF'
+# Ideas
+
+- query files with SQL
+- watch for changes
+EOF
+cat > notes/bob/reading-list.md <<'EOF'
+# Reading list
+
+- The SQLite file format
+EOF
+```
+
+(Any directory of files works with `dirsql` — the rest of this tutorial
+assumes exactly these three so your output matches ours.)
+
+Check that all three files are in place:
+
+```bash
+find notes -type f | sort
+```
+
+```
+notes/alice/ideas.md
+notes/alice/welcome.md
+notes/bob/reading-list.md
+```
+
+## 2. Query your files
+
+You wrote no configuration and no schema. Ask `dirsql` how many files are in
+this directory anyway — from inside `my-notes`, run one command:
+
+::: code-group
+
+```bash [npm]
+npx dirsql query "SELECT COUNT(*) AS files FROM './'"
+```
+
+```bash [PyPI]
+uvx dirsql query "SELECT COUNT(*) AS files FROM './'"
+```
+
+:::
+
+The first run downloads the package (`npx` asks for confirmation — answer
+`y`; `uvx` prints download progress), then prints the result:
+
+```
+[{"files":3}]
+```
+
+Three files, three rows. That one command scanned the directory, handed
+SQLite one row per file, ran your SQL, and printed the answer as JSON.
+
+There is no named table here — you never declared one. `'./'` is a
+[path-table](./reference/path-tables.md): a quoted path written where a table
+name goes. `'./'` means everything under the directory you ran the command
+in. The path *is* the query.
+
+## 3. Select some columns
+
+The response is always a JSON array of row objects, so from here on we pipe
+it through `jq` to pretty-print. Ask for two columns instead of a count:
+
+::: code-group
+
+```bash [npm]
+npx dirsql query "SELECT path, size FROM './' ORDER BY path" | jq
+```
+
+```bash [PyPI]
+uvx dirsql query "SELECT path, size FROM './' ORDER BY path" | jq
+```
+
+:::
+
+```json
+[
+  {
+    "path": "notes/alice/ideas.md",
+    "size": 52
+  },
+  {
+    "path": "notes/alice/welcome.md",
+    "size": 66
+  },
+  {
+    "path": "notes/bob/reading-list.md",
+    "size": 41
+  }
+]
+```
+
+`path` and `size` are two of the built-in file columns `dirsql` collects for
+every file — see [stat columns](./reference/columns.md#stat-columns) for the
+full list. (The `size` values are byte counts; they match the output above
+because you pasted the files exactly.)
+
+You have a working SQL database over your files, and you never left the
+command line.
+
+## 4. Declare a table
+
+A declared table fixes a shape once: you give it a name, scope it to exactly
+the files you care about, and then query it by name instead of repeating a
+path in every question. It is also the on-ramp to everything a path-table
+can't do — a named table can be kept live by the watcher, persisted across
+restarts, and given a parser that reads inside your files.
+
+Still inside `my-notes`, create a `.dirsql.toml`:
+
+```bash
+cat > .dirsql.toml <<'EOF'
+[[table]]
+ddl  = "CREATE TABLE notes (dir TEXT, basename TEXT, size INTEGER)"
+glob = "notes/**/*.md"
+EOF
+```
+
+Two keys define the table:
+
+- `glob` selects which files feed the table — every `.md` at any depth under
+  `notes/`, relative to the directory the config sits in.
+- `ddl` is ordinary `CREATE TABLE` SQL naming the columns you want to keep.
+  Each is a [stat column](./reference/columns.md#stat-columns) `dirsql`
+  computes for every file.
+
+## 5. Query the table
+
+`dirsql` does not auto-load a `.dirsql.toml` from the current directory, so
+pass it explicitly with `-c`, **after** the SQL:
+
+::: code-group
+
+```bash [npm]
+npx dirsql query "SELECT dir, basename, size FROM notes ORDER BY dir, basename" -c .dirsql.toml | jq
+```
+
+```bash [PyPI]
+uvx dirsql query "SELECT dir, basename, size FROM notes ORDER BY dir, basename" -c .dirsql.toml | jq
+```
+
+:::
+
+```json
+[
+  {
+    "basename": "ideas.md",
+    "dir": "notes/alice",
+    "size": 52
+  },
+  {
+    "basename": "welcome.md",
+    "dir": "notes/alice",
+    "size": 66
+  },
+  {
+    "basename": "reading-list.md",
+    "dir": "notes/bob",
+    "size": 41
+  }
+]
+```
+
+You queried `FROM notes` by name — no path, no glob to repeat. And because
+`dir` is a real SQL column, you can aggregate on it. Count each author's
+notes by their folder:
+
+::: code-group
+
+```bash [npm]
+npx dirsql query "SELECT dir, COUNT(*) AS notes FROM notes GROUP BY dir ORDER BY dir" -c .dirsql.toml | jq
+```
+
+```bash [PyPI]
+uvx dirsql query "SELECT dir, COUNT(*) AS notes FROM notes GROUP BY dir ORDER BY dir" -c .dirsql.toml | jq
+```
+
+:::
+
+```json
+[
+  {
+    "dir": "notes/alice",
+    "notes": 2
+  },
+  {
+    "dir": "notes/bob",
+    "notes": 1
+  }
+]
+```
+
+That's the whole loop: files in a directory, an instant query with no
+configuration, and a declared table when you want a named shape to reuse.
+
+## Where to go next
+
+- [Query files without a config](./howto/query-without-config.md) — more
+  path-table questions you can ask with no setup at all.
+- [Define tables for your files](./howto/define-tables.md) — the full
+  `[[table]]` recipe: multiple tables, ignore patterns.
+- [Extract rows from file contents](./howto/extract-from-contents.md) —
+  pull columns out of *inside* your files with an `on-file` parser.
+- [CLI](./reference/cli.md) — every flag, plus running `dirsql` as a
+  long-lived HTTP server instead of one-shot queries.
+- [HTTP API](./reference/http-api.md) — `POST /query`, plus `GET /events`,
+  a live stream of row changes as files change.
+- [SDK](./reference/sdk.md) — embed `dirsql` in a Python, Rust, or
+  TypeScript program instead of running the CLI.
+- Why is the database rebuilt from your files on every query? See
+  [how `dirsql` thinks](./explanation.md).
+```

dirsql-0.3.121/docs/howto/parse-files-into-columns.md

@@ -0,0 +1,122 @@
+# Parse your files into columns
+
+A [path-table](../reference/path-tables.md) gives you one row per file, with the
+stat columns (`path`, `size`, `mtime`, …). When the columns you actually want
+live *inside* each file, attach a parser with
+[`--on-file`](../reference/cli.md#on-file-command): prototype it inline on the
+path-table, then paste the **same** command into a config file the day you want
+a watcher, persistence, or more than one table. The parser command never
+changes across that move — that is the whole point.
+
+## 1. Start from a bare path-table
+
+You have a directory of Markdown posts with YAML frontmatter:
+
+```
+posts/hello-world.md
+posts/on-recursion.md
+```
+
+A path-table already answers questions about the files themselves:
+
+```bash
+dirsql query "SELECT basename, size FROM './posts/*.md' ORDER BY basename"
+```
+
+```json
+[{"basename":"hello-world.md","size":65},{"basename":"on-recursion.md","size":102}]
+```
+
+But `title` and `author` are inside the frontmatter, not in the stat columns.
+To get them, you need a parser.
+
+## 2. Attach a parser with `--on-file`
+
+Any program that reads one file and prints a **JSON array of row objects** on
+stdout is a parser. Here is a small one, `extract.py`, that reads a post's
+frontmatter:
+
+```python
+#!/usr/bin/env python3
+import json, re, sys
+
+text = open(sys.argv[1], encoding="utf-8").read()
+m = re.match(r"^---\n(.*?)\n---", text, re.DOTALL)
+fields = dict(
+    (k.strip(), v.strip())
+    for k, _, v in (line.partition(":") for line in (m.group(1).splitlines() if m else []))
+)
+print(json.dumps([{"title": fields.get("title"), "author": fields.get("author")}]))
+```
+
+Point the path-table at it with `--on-file`:
+
+```bash
+dirsql query "SELECT title, author FROM './posts/*.md' ORDER BY title" \
+  --on-file 'python3 extract.py {path}'
+```
+
+```json
+[{"author":"Ada Lovelace","title":"Hello World"},{"author":"Alan Turing","title":"On Recursion"}]
+```
+
+Now the parser's output *is* the table. `--on-file` runs the command once per
+matched file; `{path}` is the file's absolute path, one of the placeholders in
+the shared [`on-file` hook contract](../reference/hooks.md#on-file) (argv
+splitting, timeout, and per-file failure isolation all come from there). The
+stat columns are no longer reachable — a parser that wants the path emits it,
+since it already has `{path}`. See
+[Parsing rows with `--on-file`](../reference/path-tables.md#parsing-rows-with-on-file)
+for the full behavior.
+
+`--on-file` applies to every path-table in the query and may be given at most
+once. It is a `query`-only flag: there is no config file involved yet, so it is
+the fastest way to see whether a parser produces the rows you expect.
+
+## 3. Graduate to a config file
+
+The inline flag re-scans and re-parses every file on every query, defines one
+parser for the whole query, and forgets everything when the process exits. When
+you want a **watcher** that keeps the rows fresh, **persistence** across
+restarts, or **different parsers for different file sets**, move the parser into
+a `.dirsql.toml` [`[[table]]`](../reference/config.md#table) — and paste the
+command in verbatim:
+
+```toml
+[[table]]
+ddl     = "CREATE TABLE posts (title TEXT, author TEXT)"
+glob    = "posts/*.md"
+on-file = "python3 extract.py {path}"
+```
+
+The `on-file` value is byte-for-byte the string you passed to `--on-file`. The
+[`on-file` hook contract](../reference/hooks.md#on-file) is the same contract
+the flag used — the flag and the config key are two spellings of the same
+attachment. What you gain by moving to config is everything around the parser:
+
+```bash
+dirsql query "SELECT title, author FROM posts ORDER BY title" -c ./.dirsql.toml
+```
+
+```json
+[{"author":"Ada Lovelace","title":"Hello World"},{"author":"Alan Turing","title":"On Recursion"}]
+```
+
+Same command, same rows. The difference is what a declared table brings: it is
+indexed on build, kept fresh by the watcher, survives restarts with
+[`--persist`](./persist.md), and a config can declare
+[many tables](./define-tables.md) — each with its own `on-file` — where the flag
+gives every path-table one parser. A declared table also merges the filesystem
+facts back onto each row, so a `path TEXT` column in the DDL is populated by
+`dirsql` even though the parser did not emit it
+([precedence](../reference/columns.md#precedence)).
+
+## Going further
+
+- The full stat-vs-parsed behavior, failure isolation, and skip rules:
+  [Parsing rows with `--on-file`](../reference/path-tables.md#parsing-rows-with-on-file).
+- Starting from a declared table instead of a path-table?
+  [Extract rows from file contents](./extract-from-contents.md) covers the
+  config-first path.
+- One row per record *within* a file (JSONL, multiple frontmatter blocks): the
+  parser prints one object per row — the array length is the row count.

{dirsql-0.3.119 → dirsql-0.3.121}/docs/howto/write-a-plugin.md

@@ -217,7 +217,7 @@ Discovery is deliberately narrow. Know exactly who does what:
   your config still takes ordering precedence
   ([composing configs](../reference/config.md#composing-multiple-configs)). When
   you pass no `-c` of your own, the launcher also keeps the
-  shipped starter `files` table (an internal `--include-default`), so plugins
+  shipped starter `records` table (an internal `--include-default`), so plugins
   **add** tables rather than standing alone. Discovery is **pip/uvx only** for now — the `npx`
   launcher does not yet discover — and is switched off per invocation with
   [`--no-plugin` or `DIRSQL_NO_PLUGIN=1`](../reference/cli.md#plugins).

{dirsql-0.3.119/packages/rust → dirsql-0.3.121}/docs/index.md

@@ -34,7 +34,7 @@ db = DirSQL(
     "./my-project",
     tables=[
         Table(
-            ddl="CREATE TABLE files (name TEXT, size INTEGER, type TEXT)",
+            ddl="CREATE TABLE records (name TEXT, size INTEGER, type TEXT)",
             glob="data/*.json",
             on_file=lambda path: [json.loads(open(path, encoding="utf-8").read())],
         ),
@@ -42,7 +42,7 @@ db = DirSQL(
 )
 
 # SQL queries over your filesystem
-large = db.query("SELECT * FROM files WHERE size > 1000")
+large = db.query("SELECT * FROM records WHERE size > 1000")
 ```
 
 ```rust [Rust]
@@ -52,14 +52,14 @@ let db = DirSQL::new(
     "./my-project",
     vec![
         Table::new(
-            "CREATE TABLE files (name TEXT, size INTEGER, type TEXT)",
+            "CREATE TABLE records (name TEXT, size INTEGER, type TEXT)",
             "data/*.json",
             |path| vec![serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap()],
         ),
     ],
 )?;
 
-let large = db.query("SELECT * FROM files WHERE size > 1000")?;
+let large = db.query("SELECT * FROM records WHERE size > 1000")?;
 ```
 
 ```typescript [TypeScript]
@@ -70,14 +70,14 @@ const db = new DirSQL({
   root: './my-project',
   tables: [
     new Table({
-      ddl: 'CREATE TABLE files (name TEXT, size INTEGER, type TEXT)',
+      ddl: 'CREATE TABLE records (name TEXT, size INTEGER, type TEXT)',
       glob: 'data/*.json',
       onFile: (path) => [JSON.parse(readFileSync(path, 'utf8'))],
     }),
   ],
 });
 
-const large = await db.query('SELECT * FROM files WHERE size > 1000');
+const large = await db.query('SELECT * FROM records WHERE size > 1000');
 ```
 
 :::

{dirsql-0.3.119 → dirsql-0.3.121}/docs/reference/cli.md

@@ -168,7 +168,9 @@ timeout. The parser's output is the whole schema; the stat columns are not
 reachable on a parsed path-table. `--on-file` may be given **at most once** (a
 repeat is an error pointing at config files) and never touches config-declared
 tables. It is a `query`-only flag — server mode rejects it as an unknown
-argument.
+argument. The command string is copy-paste identical to a `[[table]]`
+`on-file` key, so an inline parser graduates to a config file unchanged — see
+[Parse your files into columns](../howto/parse-files-into-columns.md).
 
 Errors print the same diagnostic the HTTP `{"error": …}` body carries —
 config failures, SQL errors, rejected reads, hook failures, timeouts — to
@@ -183,8 +185,11 @@ stderr, with exit code `1`.
 
 ## `dirsql init`
 
-Writes a starter `.dirsql.toml` defining a catch-all `files` table, as a
-scaffold to edit:
+Writes a starter `.dirsql.toml` as a scaffold to edit. It does **not**
+duplicate the zero-config floor (`SELECT * FROM './'` already lists every file
+with no config); instead it shows the **escalation**: one named `[[table]]`
+with a glob, a schema, and a real `on-file` hook that pulls structured rows
+out of your files.
 
 ```bash
 dirsql init
@@ -225,7 +230,7 @@ installed in the same environment as `dirsql` (`pip install …`, or
 loads its fragment — its tables are queryable with zero config edits.
 Installed = active: there is no enable step and no naming convention. The
 fragment is composed *after* your own `-c` configs (so your config takes
-precedence in ordering), and the shipped starter `files` table is preserved.
+precedence in ordering), and the shipped starter `records` table is preserved.
 
 Discovery is **launcher-only** — the standalone `cargo`-installed binary does no
 discovery, and the SDKs never auto-discover (pass a plugin's config explicitly

{dirsql-0.3.119 → dirsql-0.3.121}/docs/reference/hooks.md

@@ -99,6 +99,17 @@ every watched change. The command reads the file itself and prints a JSON
 array of row objects; see [`[[table]]`](./config.md#table) for the
 row-mapping rules.
 
+The same command is attachable two ways, over the same contract: the
+`on-file` config key on a `[[table]]`, and the
+[`--on-file <command>`](./cli.md#on-file-command) flag on `dirsql query`,
+which attaches it to every [path-table](./path-tables.md#parsing-rows-with-on-file)
+in the query. The command string is identical between the two spellings — the
+flag is the inline form, the config key the declared form (see
+[Parse your files into columns](../howto/parse-files-into-columns.md)). The one
+behavioral difference is not in this contract but in the surrounding table: a
+declared `[[table]]` merges the filesystem facts onto each parsed row, while a
+parsed path-table exposes only the parser's output.
+
 | Placeholder | Value |
 |---|---|
 | `{path}` | The matched file's **absolute** path. `on-file = "extract.py {path}"` — self-sufficient from any working directory, so the command resolves it even when the config lives outside the index. |

{dirsql-0.3.119/packages/rust → dirsql-0.3.121}/docs/reference/http-api.md

@@ -27,7 +27,7 @@ The `200` response is a JSON array of row objects keyed by column name:
 ```bash
 curl -s http://localhost:7117/query \
   -H 'content-type: application/json' \
-  -d '{"sql":"SELECT COUNT(*) AS n FROM files"}' \
+  -d '{"sql":"SELECT COUNT(*) AS n FROM \'./\'"}' \
   | jq
 ```
 

{dirsql-0.3.119 → dirsql-0.3.121}/docs/reference/path-tables.md

@@ -198,9 +198,9 @@ pattern if you would rather not see them.
 Path-tables are ordinary SQLite tables once resolved, so they join freely:
 
 ```sql
-SELECT p.basename, f.size
+SELECT p.basename, d.size
 FROM './docs/*.md' AS p
-JOIN files AS f ON f.path = p.path;
+JOIN pages AS d ON d.path = p.path;
 ```
 
 A zero-match path-table is not an error — it is an empty table, and the query

{dirsql-0.3.119 → dirsql-0.3.121}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.3.119"
+version = "0.3.121"
 # abi3 note: the `abi3-py311` pyo3 feature (below) builds ONE stable-ABI
 # `cp311-abi3` wheel per platform that loads on every CPython >= 3.11
 # (matching `pyproject.toml`'s `requires-python = ">=3.11"`), instead of

{dirsql-0.3.119 → dirsql-0.3.121}/packages/python/docs/.vitepress/config.ts

@@ -41,6 +41,7 @@ export default defineConfig({
             { text: 'Define tables for your files', link: '/howto/define-tables' },
             { text: 'Derive columns from file paths', link: '/howto/columns-from-paths' },
             { text: 'Extract rows from file contents', link: '/howto/extract-from-contents' },
+            { text: 'Parse your files into columns', link: '/howto/parse-files-into-columns' },
             { text: 'Search documents by meaning', link: '/howto/search-by-meaning' },
             { text: "Skip files you don't want indexed", link: '/howto/skip-files' },
             { text: 'Load a SQLite extension', link: '/howto/load-extension' },