dirsql 0.3.43__tar.gz → 0.3.45__tar.gz

{dirsql-0.3.43 → dirsql-0.3.45}/Cargo.lock

@@ -478,11 +478,13 @@ dependencies = [
  "rusqlite",
  "serde",
  "serde_json",
+ "shlex",
  "tempfile",
  "thiserror",
  "tokio",
  "tokio-stream",
  "toml",
+ "wait-timeout",
  "walkdir",
 ]
 
@@ -499,7 +501,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.3.43"
+version = "0.3.45"
 dependencies = [
  "dirsql",
  "pyo3",

{dirsql-0.3.43 → dirsql-0.3.45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.3.43
+Version: 0.3.45
 Requires-Dist: pytest>=8 ; extra == 'dev'
 Requires-Dist: pytest-describe>=2 ; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.23 ; extra == 'dev'

{dirsql-0.3.43/packages/python → dirsql-0.3.45}/docs/cli/config.md

@@ -191,6 +191,57 @@ always filtered to the DDL's declared columns regardless. Strict mode
 applies only to keys produced by an extract callback (relevant for
 programmatic [tables](../guide/tables.md)).
 
+### Per-file commands (`on-file`)
+
+Reach for `on-file` when a table's rows come from the *contents* of each
+matched file, not just its path and stat metadata. A filesystem-fact table
+gives you one row per file; `on-file` runs a command per file that reads the
+file and emits as many rows as it likes.
+
+```toml
+[[table]]
+ddl     = "CREATE TABLE papers (paper_id TEXT, title TEXT)"
+glob    = "**/meta.json"
+on-file = "uv run python extract_papers.py {path}"
+```
+
+For every file matched by `glob`, `dirsql` runs the command. **The command
+reads the file itself and prints a JSON array of row objects on stdout**; each
+object becomes one row, its fields mapped to columns:
+
+```json
+[
+  { "paper_id": "arXiv:2401.001", "title": "On Directories" },
+  { "paper_id": "arXiv:2401.002", "title": "SQL All The Way Down" }
+]
+```
+
+Placeholders substituted into the command:
+
+| Placeholder | Value |
+|-------------|-------|
+| `{path}`    | The matched file's path **relative to the index root**. Appended automatically when the command omits it, so `extract.py` and `extract.py {path}` behave identically. |
+| `{abspath}` | The matched file's absolute path. |
+| `{root}`    | The index root directory. |
+
+Filesystem facts (stat virtuals and glob captures) are still merged onto every
+`on-file` row, so you can declare `_path`, `_basename`, `{capture}`, etc. in the
+DDL alongside the command's own columns — a column emitted by the command wins
+over a same-named filesystem fact.
+
+JSON values map to SQLite as follows: `null` → NULL; `true`/`false` → `1`/`0`;
+an integer → INTEGER, any other number → REAL; a string → TEXT; a nested array
+or object → its JSON text as TEXT.
+
+**Per-file error isolation.** If a file's command fails — a non-zero exit, a
+timeout, a spawn error, or output that isn't a JSON array of objects — that
+file is skipped (it contributes no rows) and a one-line warning naming the file
+and the error is written to stderr. One bad file never aborts the scan; the
+other files' rows are indexed normally.
+
+See [Command execution](#command-execution) for the full contract (argv
+splitting, injection safety, cwd, environment, timeout, and output framing).
+
 ### Full Example
 
 ```toml
@@ -209,3 +260,34 @@ glob = "**/index.md"
 ddl  = "CREATE TABLE logs (_path TEXT, _size INTEGER, _mtime INTEGER)"
 glob = "logs/*.csv"
 ```
+
+## Command execution
+
+Config keys that run an external command — today `on-file`, with more events to
+follow — share one execution contract:
+
+- **argv, not a shell.** The command string is split into an argv with
+  shell-like quoting (spaces separate arguments; quotes group them), but **no
+  shell is invoked** — there is no globbing, piping, `$VAR` expansion, or
+  `&&`/`;` chaining. To get those, ask for a shell explicitly:
+  `sh -c 'grep foo {path} | sort'` — the quoted script stays a single argument.
+- **Injection-safe placeholders.** Each placeholder (`{path}`, `{abspath}`,
+  `{root}`, …) is substituted into whole argv tokens, every occurrence, in a
+  single left-to-right pass. A substituted value is always exactly one argv
+  element, so a path with spaces — or untrusted content that itself contains
+  `{…}` or shell metacharacters — is inert and never re-scanned. An unknown
+  `{…}` is left literal.
+- **Working directory.** The command runs in the **config file's directory**,
+  so relative paths in the command resolve predictably regardless of where you
+  launched `dirsql`.
+- **Environment.** The command inherits `dirsql`'s environment, so tools like
+  `uvx --with …` / `npx …` resolve their dependencies as usual.
+- **Output framing.** The command's result is the **last non-empty line of
+  stdout**; any log/chatter lines above it are ignored. stderr is never data —
+  it is captured only to enrich error messages.
+- **Timeout.** Each command run is bounded by a fixed **30-second** timeout (no
+  per-table override yet); a command that exceeds it is killed and treated as a
+  failure.
+- **Errors.** A non-zero exit, a timeout, a spawn failure, or output that does
+  not parse as expected is a per-file failure: the file is skipped with a
+  stderr warning and the scan continues.

{dirsql-0.3.43 → dirsql-0.3.45}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.3.43"
+version = "0.3.45"
 edition.workspace = true
 publish = false
 readme = "README.md"

{dirsql-0.3.43 → dirsql-0.3.45/packages/python}/docs/cli/config.md

@@ -191,6 +191,57 @@ always filtered to the DDL's declared columns regardless. Strict mode
 applies only to keys produced by an extract callback (relevant for
 programmatic [tables](../guide/tables.md)).
 
+### Per-file commands (`on-file`)
+
+Reach for `on-file` when a table's rows come from the *contents* of each
+matched file, not just its path and stat metadata. A filesystem-fact table
+gives you one row per file; `on-file` runs a command per file that reads the
+file and emits as many rows as it likes.
+
+```toml
+[[table]]
+ddl     = "CREATE TABLE papers (paper_id TEXT, title TEXT)"
+glob    = "**/meta.json"
+on-file = "uv run python extract_papers.py {path}"
+```
+
+For every file matched by `glob`, `dirsql` runs the command. **The command
+reads the file itself and prints a JSON array of row objects on stdout**; each
+object becomes one row, its fields mapped to columns:
+
+```json
+[
+  { "paper_id": "arXiv:2401.001", "title": "On Directories" },
+  { "paper_id": "arXiv:2401.002", "title": "SQL All The Way Down" }
+]
+```
+
+Placeholders substituted into the command:
+
+| Placeholder | Value |
+|-------------|-------|
+| `{path}`    | The matched file's path **relative to the index root**. Appended automatically when the command omits it, so `extract.py` and `extract.py {path}` behave identically. |
+| `{abspath}` | The matched file's absolute path. |
+| `{root}`    | The index root directory. |
+
+Filesystem facts (stat virtuals and glob captures) are still merged onto every
+`on-file` row, so you can declare `_path`, `_basename`, `{capture}`, etc. in the
+DDL alongside the command's own columns — a column emitted by the command wins
+over a same-named filesystem fact.
+
+JSON values map to SQLite as follows: `null` → NULL; `true`/`false` → `1`/`0`;
+an integer → INTEGER, any other number → REAL; a string → TEXT; a nested array
+or object → its JSON text as TEXT.
+
+**Per-file error isolation.** If a file's command fails — a non-zero exit, a
+timeout, a spawn error, or output that isn't a JSON array of objects — that
+file is skipped (it contributes no rows) and a one-line warning naming the file
+and the error is written to stderr. One bad file never aborts the scan; the
+other files' rows are indexed normally.
+
+See [Command execution](#command-execution) for the full contract (argv
+splitting, injection safety, cwd, environment, timeout, and output framing).
+
 ### Full Example
 
 ```toml
@@ -209,3 +260,34 @@ glob = "**/index.md"
 ddl  = "CREATE TABLE logs (_path TEXT, _size INTEGER, _mtime INTEGER)"
 glob = "logs/*.csv"
 ```
+
+## Command execution
+
+Config keys that run an external command — today `on-file`, with more events to
+follow — share one execution contract:
+
+- **argv, not a shell.** The command string is split into an argv with
+  shell-like quoting (spaces separate arguments; quotes group them), but **no
+  shell is invoked** — there is no globbing, piping, `$VAR` expansion, or
+  `&&`/`;` chaining. To get those, ask for a shell explicitly:
+  `sh -c 'grep foo {path} | sort'` — the quoted script stays a single argument.
+- **Injection-safe placeholders.** Each placeholder (`{path}`, `{abspath}`,
+  `{root}`, …) is substituted into whole argv tokens, every occurrence, in a
+  single left-to-right pass. A substituted value is always exactly one argv
+  element, so a path with spaces — or untrusted content that itself contains
+  `{…}` or shell metacharacters — is inert and never re-scanned. An unknown
+  `{…}` is left literal.
+- **Working directory.** The command runs in the **config file's directory**,
+  so relative paths in the command resolve predictably regardless of where you
+  launched `dirsql`.
+- **Environment.** The command inherits `dirsql`'s environment, so tools like
+  `uvx --with …` / `npx …` resolve their dependencies as usual.
+- **Output framing.** The command's result is the **last non-empty line of
+  stdout**; any log/chatter lines above it are ignored. stderr is never data —
+  it is captured only to enrich error messages.
+- **Timeout.** Each command run is bounded by a fixed **30-second** timeout (no
+  per-table override yet); a command that exceeds it is killed and treated as a
+  failure.
+- **Errors.** A non-zero exit, a timeout, a spawn failure, or output that does
+  not parse as expected is a per-file failure: the file is skipped with a
+  stderr warning and the scan continues.

{dirsql-0.3.43 → dirsql-0.3.45}/packages/rust/Cargo.toml

@@ -60,10 +60,16 @@ regex.workspace = true
 rusqlite = { version = "0.34", features = ["bundled", "load_extension"] }
 serde.workspace = true
 serde_json.workspace = true
+# Split a command *template* into argv with shell-like quoting (so
+# `sh -c '...'` keeps its script as one arg) WITHOUT invoking a shell.
+shlex = "1"
 thiserror.workspace = true
 tokio.workspace = true
 tokio-stream = { version = "0.1", features = ["sync"], optional = true }
 toml = "0.8"
+# Enforce a per-command timeout: wait for the child up to a deadline,
+# then kill it. No async runtime required in the core.
+wait-timeout = "0.2"
 walkdir = "2"
 
 [[bin]]

{dirsql-0.3.43 → dirsql-0.3.45}/packages/rust/docs/cli/config.md

@@ -191,6 +191,57 @@ always filtered to the DDL's declared columns regardless. Strict mode
 applies only to keys produced by an extract callback (relevant for
 programmatic [tables](../guide/tables.md)).
 
+### Per-file commands (`on-file`)
+
+Reach for `on-file` when a table's rows come from the *contents* of each
+matched file, not just its path and stat metadata. A filesystem-fact table
+gives you one row per file; `on-file` runs a command per file that reads the
+file and emits as many rows as it likes.
+
+```toml
+[[table]]
+ddl     = "CREATE TABLE papers (paper_id TEXT, title TEXT)"
+glob    = "**/meta.json"
+on-file = "uv run python extract_papers.py {path}"
+```
+
+For every file matched by `glob`, `dirsql` runs the command. **The command
+reads the file itself and prints a JSON array of row objects on stdout**; each
+object becomes one row, its fields mapped to columns:
+
+```json
+[
+  { "paper_id": "arXiv:2401.001", "title": "On Directories" },
+  { "paper_id": "arXiv:2401.002", "title": "SQL All The Way Down" }
+]
+```
+
+Placeholders substituted into the command:
+
+| Placeholder | Value |
+|-------------|-------|
+| `{path}`    | The matched file's path **relative to the index root**. Appended automatically when the command omits it, so `extract.py` and `extract.py {path}` behave identically. |
+| `{abspath}` | The matched file's absolute path. |
+| `{root}`    | The index root directory. |
+
+Filesystem facts (stat virtuals and glob captures) are still merged onto every
+`on-file` row, so you can declare `_path`, `_basename`, `{capture}`, etc. in the
+DDL alongside the command's own columns — a column emitted by the command wins
+over a same-named filesystem fact.
+
+JSON values map to SQLite as follows: `null` → NULL; `true`/`false` → `1`/`0`;
+an integer → INTEGER, any other number → REAL; a string → TEXT; a nested array
+or object → its JSON text as TEXT.
+
+**Per-file error isolation.** If a file's command fails — a non-zero exit, a
+timeout, a spawn error, or output that isn't a JSON array of objects — that
+file is skipped (it contributes no rows) and a one-line warning naming the file
+and the error is written to stderr. One bad file never aborts the scan; the
+other files' rows are indexed normally.
+
+See [Command execution](#command-execution) for the full contract (argv
+splitting, injection safety, cwd, environment, timeout, and output framing).
+
 ### Full Example
 
 ```toml
@@ -209,3 +260,34 @@ glob = "**/index.md"
 ddl  = "CREATE TABLE logs (_path TEXT, _size INTEGER, _mtime INTEGER)"
 glob = "logs/*.csv"
 ```
+
+## Command execution
+
+Config keys that run an external command — today `on-file`, with more events to
+follow — share one execution contract:
+
+- **argv, not a shell.** The command string is split into an argv with
+  shell-like quoting (spaces separate arguments; quotes group them), but **no
+  shell is invoked** — there is no globbing, piping, `$VAR` expansion, or
+  `&&`/`;` chaining. To get those, ask for a shell explicitly:
+  `sh -c 'grep foo {path} | sort'` — the quoted script stays a single argument.
+- **Injection-safe placeholders.** Each placeholder (`{path}`, `{abspath}`,
+  `{root}`, …) is substituted into whole argv tokens, every occurrence, in a
+  single left-to-right pass. A substituted value is always exactly one argv
+  element, so a path with spaces — or untrusted content that itself contains
+  `{…}` or shell metacharacters — is inert and never re-scanned. An unknown
+  `{…}` is left literal.
+- **Working directory.** The command runs in the **config file's directory**,
+  so relative paths in the command resolve predictably regardless of where you
+  launched `dirsql`.
+- **Environment.** The command inherits `dirsql`'s environment, so tools like
+  `uvx --with …` / `npx …` resolve their dependencies as usual.
+- **Output framing.** The command's result is the **last non-empty line of
+  stdout**; any log/chatter lines above it are ignored. stderr is never data —
+  it is captured only to enrich error messages.
+- **Timeout.** Each command run is bounded by a fixed **30-second** timeout (no
+  per-table override yet); a command that exceeds it is killed and treated as a
+  failure.
+- **Errors.** A non-zero exit, a timeout, a spawn failure, or output that does
+  not parse as expected is a per-file failure: the file is skipped with a
+  stderr warning and the scan continues.