dirsql 0.3.96__tar.gz → 0.3.98__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dirsql-0.3.96 → dirsql-0.3.98}/Cargo.lock +1 -1
- {dirsql-0.3.96 → dirsql-0.3.98}/PKG-INFO +1 -1
- {dirsql-0.3.96 → dirsql-0.3.98}/dirsql/_dirsql.pyi +1 -1
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/howto/embed.md +6 -6
- {dirsql-0.3.96/packages/python → dirsql-0.3.98}/docs/howto/extract-from-contents.md +1 -1
- {dirsql-0.3.96/packages/rust → dirsql-0.3.98}/docs/index.md +3 -3
- {dirsql-0.3.96/packages/rust → dirsql-0.3.98}/docs/reference/cli.md +1 -1
- {dirsql-0.3.96/packages/rust → dirsql-0.3.98}/docs/reference/columns.md +3 -3
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/reference/config.md +34 -2
- {dirsql-0.3.96/packages/rust → dirsql-0.3.98}/docs/reference/sdk.md +8 -8
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/Cargo.toml +1 -1
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/howto/embed.md +6 -6
- {dirsql-0.3.96/packages/rust → dirsql-0.3.98/packages/python}/docs/howto/extract-from-contents.md +1 -1
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/index.md +3 -3
- {dirsql-0.3.96 → dirsql-0.3.98/packages/python}/docs/reference/cli.md +1 -1
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/reference/columns.md +3 -3
- {dirsql-0.3.96/packages/rust → dirsql-0.3.98/packages/python}/docs/reference/config.md +34 -2
- {dirsql-0.3.96 → dirsql-0.3.98/packages/python}/docs/reference/sdk.md +8 -8
- dirsql-0.3.98/packages/python/e2e-attestations/claude-open-issues-review-le8hm5-red-570.json +7 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/src/lib.rs +19 -19
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/docs/howto/embed.md +6 -6
- {dirsql-0.3.96 → dirsql-0.3.98/packages/rust}/docs/howto/extract-from-contents.md +1 -1
- {dirsql-0.3.96 → dirsql-0.3.98/packages/rust}/docs/index.md +3 -3
- {dirsql-0.3.96/packages/python → dirsql-0.3.98/packages/rust}/docs/reference/cli.md +1 -1
- {dirsql-0.3.96 → dirsql-0.3.98/packages/rust}/docs/reference/columns.md +3 -3
- {dirsql-0.3.96/packages/python → dirsql-0.3.98/packages/rust}/docs/reference/config.md +34 -2
- {dirsql-0.3.96/packages/python → dirsql-0.3.98/packages/rust}/docs/reference/sdk.md +8 -8
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/src/bin/dirsql.rs +110 -73
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/src/config.rs +1 -1
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/src/db.rs +1 -1
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/src/lib.rs +41 -41
- dirsql-0.3.96/packages/python/e2e-attestations/claude-github-issue-580-d0jd9u.json +0 -7
- dirsql-0.3.96/packages/python/e2e-attestations/claude-open-issues-review-le8hm5-red-546.json +0 -7
- dirsql-0.3.96/packages/python/e2e-attestations/claude-open-issues-review-le8hm5-red-553.json +0 -7
- {dirsql-0.3.96 → dirsql-0.3.98}/Cargo.toml +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/README.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/dirsql/__init__.py +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/dirsql/_async.py +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/dirsql/cli/__init__.py +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/dirsql/cli/binary_path.py +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/dirsql/cli/is_windows.py +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/dirsql/cli/main.py +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/dirsql/cli/resolve_config_extensions.py +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/dirsql/py.typed +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/dirsql/resolve_config_extensions.py +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/dirsql/resolve_extension.py +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/.claude/CLAUDE.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/.vitepress/config.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/.vitepress/theme/index.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/.vitepress/theme/lang.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/AGENTS.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/explanation.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/getting-started.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/howto/columns-from-paths.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/howto/define-tables.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/howto/load-extension.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/howto/persist.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/howto/react-to-changes.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/howto/search-by-meaning.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/howto/skip-files.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/migrations.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/package.json +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/playwright.config.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/pnpm-lock.yaml +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/pnpm-workspace.yaml +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/reference/hooks.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/reference/http-api.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/tests/integration/home.spec.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/tests/integration/language-flag.spec.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/tests/integration/sidebar.spec.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/tests/unit/config.test.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/tests/unit/lang.test.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/docs/vitest.config.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/README.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/conftest.py +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/.claude/CLAUDE.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/.vitepress/config.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/.vitepress/theme/index.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/.vitepress/theme/lang.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/AGENTS.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/explanation.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/getting-started.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/howto/columns-from-paths.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/howto/define-tables.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/howto/load-extension.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/howto/persist.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/howto/react-to-changes.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/howto/search-by-meaning.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/howto/skip-files.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/migrations.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/package.json +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/playwright.config.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/pnpm-lock.yaml +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/pnpm-workspace.yaml +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/reference/hooks.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/reference/http-api.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/tests/integration/home.spec.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/tests/integration/language-flag.spec.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/tests/integration/sidebar.spec.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/tests/unit/config.test.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/tests/unit/lang.test.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/docs/vitest.config.ts +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/tests/__init__.py +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/tests/conftest.py +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/tests/e2e/__init__.py +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/tests/integration/__init__.py +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/tests/integration/binding/__init__.py +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/python/tests/integration/hermetic/__init__.py +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/Cargo.toml +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/README.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/benches/db_bench.rs +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/benches/differ_bench.rs +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/benches/matcher_bench.rs +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/benches/scanner_bench.rs +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/docs/explanation.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/docs/getting-started.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/docs/howto/columns-from-paths.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/docs/howto/define-tables.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/docs/howto/load-extension.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/docs/howto/persist.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/docs/howto/react-to-changes.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/docs/howto/search-by-meaning.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/docs/howto/skip-files.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/docs/migrations.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/docs/reference/hooks.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/docs/reference/http-api.md +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/src/cli/execute.rs +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/src/cli/init.rs +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/src/cli/mod.rs +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/src/cli/router.rs +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/src/cli/serialize.rs +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/src/cli/server.rs +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/src/command.rs +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/src/default_config.toml +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/src/differ.rs +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/src/matcher.rs +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/src/persist.rs +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/src/scanner.rs +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/packages/rust/src/watcher.rs +0 -0
- {dirsql-0.3.96 → dirsql-0.3.98}/pyproject.toml +0 -0
|
@@ -33,7 +33,7 @@ comments/t1/c2.json # {"body": "nice post", "author": "bob"}
|
|
|
33
33
|
comments/t2/c1.json # {"body": "following up", "author": "alice"}
|
|
34
34
|
```
|
|
35
35
|
|
|
36
|
-
Unlike a config-file table, a programmatic table takes an `
|
|
36
|
+
Unlike a config-file table, a programmatic table takes an `on_file`
|
|
37
37
|
callback — your code reads each matched file and returns its rows, with
|
|
38
38
|
[glob captures and stat columns](../reference/columns.md) merged on
|
|
39
39
|
automatically (here, `{thread}` from the path):
|
|
@@ -47,7 +47,7 @@ import json
|
|
|
47
47
|
from dirsql import DirSQL, Table
|
|
48
48
|
|
|
49
49
|
|
|
50
|
-
def
|
|
50
|
+
def on_file(path: str) -> list[dict]:
|
|
51
51
|
with open(path, encoding="utf-8") as f:
|
|
52
52
|
return [json.load(f)]
|
|
53
53
|
|
|
@@ -59,7 +59,7 @@ async def main() -> None:
|
|
|
59
59
|
Table(
|
|
60
60
|
ddl="CREATE TABLE comments (thread TEXT, author TEXT, body TEXT)",
|
|
61
61
|
glob="comments/{thread}/*.json",
|
|
62
|
-
|
|
62
|
+
on_file=on_file,
|
|
63
63
|
)
|
|
64
64
|
],
|
|
65
65
|
)
|
|
@@ -81,7 +81,7 @@ const db = new DirSQL({
|
|
|
81
81
|
{
|
|
82
82
|
ddl: "CREATE TABLE comments (thread TEXT, author TEXT, body TEXT)",
|
|
83
83
|
glob: "comments/{thread}/*.json",
|
|
84
|
-
|
|
84
|
+
onFile: (path) => [JSON.parse(readFileSync(path, "utf8"))],
|
|
85
85
|
},
|
|
86
86
|
],
|
|
87
87
|
});
|
|
@@ -99,7 +99,7 @@ use std::collections::HashMap;
|
|
|
99
99
|
|
|
100
100
|
use dirsql::{DirSQL, Table, Value};
|
|
101
101
|
|
|
102
|
-
fn
|
|
102
|
+
fn on_file(path: &str) -> Vec<HashMap<String, Value>> {
|
|
103
103
|
let raw = std::fs::read_to_string(path).unwrap_or_default();
|
|
104
104
|
let Ok(serde_json::Value::Object(obj)) = serde_json::from_str(&raw) else {
|
|
105
105
|
return vec![];
|
|
@@ -119,7 +119,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|
|
119
119
|
.table(Table::new(
|
|
120
120
|
"CREATE TABLE comments (thread TEXT, author TEXT, body TEXT)",
|
|
121
121
|
"comments/{thread}/*.json",
|
|
122
|
-
|
|
122
|
+
on_file,
|
|
123
123
|
))
|
|
124
124
|
.build()?;
|
|
125
125
|
|
|
@@ -79,6 +79,6 @@ SQLite value mapping is under
|
|
|
79
79
|
file. If it is expensive, [keep the index across restarts](./persist.md).
|
|
80
80
|
- The flagship use of `on-file` — computing embeddings — is
|
|
81
81
|
[Search documents by meaning](./search-by-meaning.md).
|
|
82
|
-
- Embedding `dirsql` in a program instead? The SDK's `
|
|
82
|
+
- Embedding `dirsql` in a program instead? The SDK's `on_file` callback
|
|
83
83
|
fills the same role in-process — see
|
|
84
84
|
[Embed `dirsql` in your application](./embed.md).
|
|
@@ -20,7 +20,7 @@ But querying across many files is slow.
|
|
|
20
20
|
|
|
21
21
|
## Solution
|
|
22
22
|
|
|
23
|
-
`dirsql` bridges this gap. The filesystem remains the source of truth, but you get SQL queries and real-time change events for free. Define tables with glob patterns and
|
|
23
|
+
`dirsql` bridges this gap. The filesystem remains the source of truth, but you get SQL queries and real-time change events for free. Define tables with glob patterns and on-file callbacks, and `dirsql` handles the rest.
|
|
24
24
|
|
|
25
25
|
::: code-group
|
|
26
26
|
|
|
@@ -34,7 +34,7 @@ db = DirSQL(
|
|
|
34
34
|
Table(
|
|
35
35
|
ddl="CREATE TABLE files (name TEXT, size INTEGER, type TEXT)",
|
|
36
36
|
glob="data/*.json",
|
|
37
|
-
|
|
37
|
+
on_file=lambda path: [json.loads(open(path, encoding="utf-8").read())],
|
|
38
38
|
),
|
|
39
39
|
],
|
|
40
40
|
)
|
|
@@ -70,7 +70,7 @@ const db = new DirSQL({
|
|
|
70
70
|
new Table({
|
|
71
71
|
ddl: 'CREATE TABLE files (name TEXT, size INTEGER, type TEXT)',
|
|
72
72
|
glob: 'data/*.json',
|
|
73
|
-
|
|
73
|
+
onFile: (path) => [JSON.parse(readFileSync(path, 'utf8'))],
|
|
74
74
|
}),
|
|
75
75
|
],
|
|
76
76
|
});
|
|
@@ -45,7 +45,7 @@ requests, closes open `/events` streams, and exits.
|
|
|
45
45
|
|
|
46
46
|
| Flag | Default | Description |
|
|
47
47
|
|---|---|---|
|
|
48
|
-
| `-c, --config <path>` | `./.dirsql.toml` | Path to
|
|
48
|
+
| `-c, --config <path>` | `./.dirsql.toml` | Path to a [config file](./config.md). **Repeatable** (`-c a -c b`): the configs load and merge in argv order — see [Composing multiple configs](./config.md#composing-multiple-configs). The index is always rooted at the **invocation directory** (the current working directory), regardless of where a config lives — so `--config /elsewhere/.dirsql.toml` still indexes the directory you ran `dirsql` from. With none given, `./.dirsql.toml` is used; when it does not exist, the server runs in [zero-config mode](#zero-config-mode). |
|
|
49
49
|
| `--host <addr>` | `localhost` | Bind address. |
|
|
50
50
|
| `--port <n>` | `7117` | TCP port to bind. |
|
|
51
51
|
| `--persist [<path>]` | off | Keep the SQLite index on disk between runs so a restart only re-parses files that actually changed. Bare `--persist` caches at `<root>/.dirsql/cache.db`; `--persist <path>` caches at `<path>`. Off by default (the index is ephemeral). Global — also honored by [`dirsql query`](#dirsql-query). See [Keep the index across restarts](../howto/persist.md). |
|
|
@@ -18,7 +18,7 @@ Declaring them requires nothing else. The names below aren't a protected or
|
|
|
18
18
|
enforced namespace — nothing stops you from declaring a column with one of
|
|
19
19
|
these names for an unrelated purpose, in which case dirsql's computed value
|
|
20
20
|
lands there like any other fact (unless your own row source — an `on-file`
|
|
21
|
-
command or SDK `
|
|
21
|
+
command or SDK `on_file` callback — supplies its own value for that name;
|
|
22
22
|
see [Precedence](#precedence)).
|
|
23
23
|
|
|
24
24
|
## Stat columns
|
|
@@ -71,8 +71,8 @@ A file at `_comments/abc123/2024-05-05.jsonl` produces a row with
|
|
|
71
71
|
## Precedence
|
|
72
72
|
|
|
73
73
|
Values produced by a table's own row source — an `on-file` command's JSON
|
|
74
|
-
output or an SDK `
|
|
75
|
-
auto-injected facts of the same name. An
|
|
74
|
+
output or an SDK `on_file` callback's return value — **win** over
|
|
75
|
+
auto-injected facts of the same name. An `on_file` callback that explicitly emits
|
|
76
76
|
`path` is honored.
|
|
77
77
|
|
|
78
78
|
Injection order per row: stat columns first, then glob captures, then
|
|
@@ -98,13 +98,13 @@ per-file command.
|
|
|
98
98
|
|---|---|---|
|
|
99
99
|
| `ddl` | yes | A SQLite `CREATE TABLE` statement. The table name is parsed from it. Only columns declared here are populated; auto-injected facts not in the DDL are dropped. |
|
|
100
100
|
| `glob` | yes | Glob pattern matched against root-relative paths. May contain `{name}` [capture segments](./columns.md#glob-captures). Every table whose glob matches a file receives that file's rows — a file can populate multiple tables. |
|
|
101
|
-
| `strict` | no (default `false`) | When `true`, rows whose keys do not exactly match the declared columns are rejected with an error: extra keys error, and every declared column must be supplied (by the command/
|
|
101
|
+
| `strict` | no (default `false`) | When `true`, rows whose keys do not exactly match the declared columns are rejected with an error: extra keys error, and every declared column must be supplied (by the command/on-file output, a glob capture, or a stat column). When `false`, extra keys are dropped and missing columns become `NULL`. |
|
|
102
102
|
| `on-file` | no | A command run once per matched file; its stdout (a JSON array of row objects) becomes the file's rows. Must be non-empty. See [Command hooks](./hooks.md#on-file). |
|
|
103
103
|
|
|
104
104
|
Without `on-file`, a table produces exactly one row per matched file, built
|
|
105
105
|
entirely from filesystem facts. Content interpretation (frontmatter, JSON
|
|
106
106
|
fields, CSV parsing) is out of scope for plain config tables — use
|
|
107
|
-
`on-file`, or a programmatic [SDK table](./sdk.md#table) with an `
|
|
107
|
+
`on-file`, or a programmatic [SDK table](./sdk.md#table) with an `on_file`
|
|
108
108
|
callback.
|
|
109
109
|
|
|
110
110
|
```toml
|
|
@@ -131,6 +131,38 @@ by the command wins over a same-named fact. Output that is not a JSON array
|
|
|
131
131
|
of objects is a per-file failure: the file is skipped with a stderr warning
|
|
132
132
|
and the scan continues (see [failure semantics](./hooks.md#failure-semantics)).
|
|
133
133
|
|
|
134
|
+
## Composing multiple configs
|
|
135
|
+
|
|
136
|
+
Pass [`-c`/`--config`](./cli.md#flags) more than once to compose several config
|
|
137
|
+
files — a shared team config plus local overrides, or a plugin's config
|
|
138
|
+
alongside your own:
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
dirsql -c ./.dirsql.toml -c ~/team/embeddings.toml -c ./local.toml
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
The configs load and merge in **argv order**:
|
|
145
|
+
|
|
146
|
+
- **`[[table]]`, `ignore`, and `[[dirsql.extension]]` entries accumulate** across
|
|
147
|
+
all configs, in order.
|
|
148
|
+
- **Each config's `on-file`, `pre-query`, and `post-query` hooks run from that
|
|
149
|
+
config file's own directory**, under that config's own
|
|
150
|
+
[`hook-timeout`](#dirsql-keys) — so a relative command like
|
|
151
|
+
`on-file = "sh ./extract.sh"` resolves against the config that declared it,
|
|
152
|
+
wherever it lives.
|
|
153
|
+
- **`pre-query` / `post-query` hooks chain FIFO**: the request body flows through
|
|
154
|
+
each `pre-query` stage in order to the final SQL, and the result rows flow
|
|
155
|
+
through each `post-query` stage to the response. See the
|
|
156
|
+
[hook contract](./hooks.md).
|
|
157
|
+
- Each config is **validated on its own** (the [parse errors](#parse-errors)
|
|
158
|
+
below apply per file). There is no cross-file merge validation, with one
|
|
159
|
+
structural exception: **two configs defining a table of the same name is an
|
|
160
|
+
error**, naming the table.
|
|
161
|
+
|
|
162
|
+
The index [root](./cli.md#flags) is the invocation directory regardless of where
|
|
163
|
+
any config lives. With no `-c`, `./.dirsql.toml` is used; a single `-c` behaves
|
|
164
|
+
exactly as before.
|
|
165
|
+
|
|
134
166
|
## Parse errors
|
|
135
167
|
|
|
136
168
|
Loading fails (the CLI enters [degraded mode](./cli.md#degraded-mode); the
|
|
@@ -296,19 +296,19 @@ rather than waiting — an intentional, language-idiomatic difference.
|
|
|
296
296
|
::: code-group
|
|
297
297
|
|
|
298
298
|
```python [Python]
|
|
299
|
-
Table(*, ddl: str, glob: str,
|
|
299
|
+
Table(*, ddl: str, glob: str, on_file: Callable[[str], list[dict]], strict: bool = False)
|
|
300
300
|
```
|
|
301
301
|
|
|
302
302
|
```typescript [TypeScript]
|
|
303
|
-
new Table({ ddl, glob,
|
|
303
|
+
new Table({ ddl, glob, onFile, strict? })
|
|
304
304
|
// or a plain object — TableDef and Table are interchangeable:
|
|
305
|
-
{ ddl: string, glob: string,
|
|
305
|
+
{ ddl: string, glob: string, onFile: (path: string) => Record<string, unknown>[], strict?: boolean }
|
|
306
306
|
```
|
|
307
307
|
|
|
308
308
|
```rust [Rust]
|
|
309
|
-
Table::new(ddl, glob,
|
|
310
|
-
Table::try_new(ddl, glob,
|
|
311
|
-
Table::strict(ddl, glob,
|
|
309
|
+
Table::new(ddl, glob, on_file) // on_file: Fn(&str) -> Vec<Row>, infallible
|
|
310
|
+
Table::try_new(ddl, glob, on_file) // on_file: Fn(&str) -> Result<Vec<Row>, _>
|
|
311
|
+
Table::strict(ddl, glob, on_file) // Table::new with strict = true
|
|
312
312
|
```
|
|
313
313
|
|
|
314
314
|
:::
|
|
@@ -321,7 +321,7 @@ Maps files to table rows.
|
|
|
321
321
|
`{name}` [captures](./columns.md#glob-captures). Every table whose glob
|
|
322
322
|
matches a file receives that file's rows — a file can populate multiple
|
|
323
323
|
tables.
|
|
324
|
-
- `
|
|
324
|
+
- `on_file` (`on_file` / `onFile`) — Callback receiving the matched file's full path (the root
|
|
325
325
|
joined with the file's relative path — absolute when `root` is absolute)
|
|
326
326
|
and returning the rows that file contributes. `dirsql` never reads file
|
|
327
327
|
contents itself; a callback that needs the body reads the path. Return
|
|
@@ -356,7 +356,7 @@ relative to the root. Errors never terminate the stream.
|
|
|
356
356
|
|
|
357
357
|
## Value types
|
|
358
358
|
|
|
359
|
-
An `
|
|
359
|
+
An `on_file` callback (and the SQLite columns it feeds) accepts:
|
|
360
360
|
|
|
361
361
|
| SQLite column | Python value | TypeScript value | Rust `Value` |
|
|
362
362
|
|---|---|---|---|
|
|
@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
|
|
|
4
4
|
# pypi/maturin handler can rewrite it via `write-version` before
|
|
5
5
|
# `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
|
|
6
6
|
# and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.98"
|
|
8
8
|
# abi3 note: the `abi3-py311` pyo3 feature (below) builds ONE stable-ABI
|
|
9
9
|
# `cp311-abi3` wheel per platform that loads on every CPython >= 3.11
|
|
10
10
|
# (matching `pyproject.toml`'s `requires-python = ">=3.11"`), instead of
|
|
@@ -33,7 +33,7 @@ comments/t1/c2.json # {"body": "nice post", "author": "bob"}
|
|
|
33
33
|
comments/t2/c1.json # {"body": "following up", "author": "alice"}
|
|
34
34
|
```
|
|
35
35
|
|
|
36
|
-
Unlike a config-file table, a programmatic table takes an `
|
|
36
|
+
Unlike a config-file table, a programmatic table takes an `on_file`
|
|
37
37
|
callback — your code reads each matched file and returns its rows, with
|
|
38
38
|
[glob captures and stat columns](../reference/columns.md) merged on
|
|
39
39
|
automatically (here, `{thread}` from the path):
|
|
@@ -47,7 +47,7 @@ import json
|
|
|
47
47
|
from dirsql import DirSQL, Table
|
|
48
48
|
|
|
49
49
|
|
|
50
|
-
def
|
|
50
|
+
def on_file(path: str) -> list[dict]:
|
|
51
51
|
with open(path, encoding="utf-8") as f:
|
|
52
52
|
return [json.load(f)]
|
|
53
53
|
|
|
@@ -59,7 +59,7 @@ async def main() -> None:
|
|
|
59
59
|
Table(
|
|
60
60
|
ddl="CREATE TABLE comments (thread TEXT, author TEXT, body TEXT)",
|
|
61
61
|
glob="comments/{thread}/*.json",
|
|
62
|
-
|
|
62
|
+
on_file=on_file,
|
|
63
63
|
)
|
|
64
64
|
],
|
|
65
65
|
)
|
|
@@ -81,7 +81,7 @@ const db = new DirSQL({
|
|
|
81
81
|
{
|
|
82
82
|
ddl: "CREATE TABLE comments (thread TEXT, author TEXT, body TEXT)",
|
|
83
83
|
glob: "comments/{thread}/*.json",
|
|
84
|
-
|
|
84
|
+
onFile: (path) => [JSON.parse(readFileSync(path, "utf8"))],
|
|
85
85
|
},
|
|
86
86
|
],
|
|
87
87
|
});
|
|
@@ -99,7 +99,7 @@ use std::collections::HashMap;
|
|
|
99
99
|
|
|
100
100
|
use dirsql::{DirSQL, Table, Value};
|
|
101
101
|
|
|
102
|
-
fn
|
|
102
|
+
fn on_file(path: &str) -> Vec<HashMap<String, Value>> {
|
|
103
103
|
let raw = std::fs::read_to_string(path).unwrap_or_default();
|
|
104
104
|
let Ok(serde_json::Value::Object(obj)) = serde_json::from_str(&raw) else {
|
|
105
105
|
return vec![];
|
|
@@ -119,7 +119,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|
|
119
119
|
.table(Table::new(
|
|
120
120
|
"CREATE TABLE comments (thread TEXT, author TEXT, body TEXT)",
|
|
121
121
|
"comments/{thread}/*.json",
|
|
122
|
-
|
|
122
|
+
on_file,
|
|
123
123
|
))
|
|
124
124
|
.build()?;
|
|
125
125
|
|
{dirsql-0.3.96/packages/rust → dirsql-0.3.98/packages/python}/docs/howto/extract-from-contents.md
RENAMED
|
@@ -79,6 +79,6 @@ SQLite value mapping is under
|
|
|
79
79
|
file. If it is expensive, [keep the index across restarts](./persist.md).
|
|
80
80
|
- The flagship use of `on-file` — computing embeddings — is
|
|
81
81
|
[Search documents by meaning](./search-by-meaning.md).
|
|
82
|
-
- Embedding `dirsql` in a program instead? The SDK's `
|
|
82
|
+
- Embedding `dirsql` in a program instead? The SDK's `on_file` callback
|
|
83
83
|
fills the same role in-process — see
|
|
84
84
|
[Embed `dirsql` in your application](./embed.md).
|
|
@@ -20,7 +20,7 @@ But querying across many files is slow.
|
|
|
20
20
|
|
|
21
21
|
## Solution
|
|
22
22
|
|
|
23
|
-
`dirsql` bridges this gap. The filesystem remains the source of truth, but you get SQL queries and real-time change events for free. Define tables with glob patterns and
|
|
23
|
+
`dirsql` bridges this gap. The filesystem remains the source of truth, but you get SQL queries and real-time change events for free. Define tables with glob patterns and on-file callbacks, and `dirsql` handles the rest.
|
|
24
24
|
|
|
25
25
|
::: code-group
|
|
26
26
|
|
|
@@ -34,7 +34,7 @@ db = DirSQL(
|
|
|
34
34
|
Table(
|
|
35
35
|
ddl="CREATE TABLE files (name TEXT, size INTEGER, type TEXT)",
|
|
36
36
|
glob="data/*.json",
|
|
37
|
-
|
|
37
|
+
on_file=lambda path: [json.loads(open(path, encoding="utf-8").read())],
|
|
38
38
|
),
|
|
39
39
|
],
|
|
40
40
|
)
|
|
@@ -70,7 +70,7 @@ const db = new DirSQL({
|
|
|
70
70
|
new Table({
|
|
71
71
|
ddl: 'CREATE TABLE files (name TEXT, size INTEGER, type TEXT)',
|
|
72
72
|
glob: 'data/*.json',
|
|
73
|
-
|
|
73
|
+
onFile: (path) => [JSON.parse(readFileSync(path, 'utf8'))],
|
|
74
74
|
}),
|
|
75
75
|
],
|
|
76
76
|
});
|
|
@@ -45,7 +45,7 @@ requests, closes open `/events` streams, and exits.
|
|
|
45
45
|
|
|
46
46
|
| Flag | Default | Description |
|
|
47
47
|
|---|---|---|
|
|
48
|
-
| `-c, --config <path>` | `./.dirsql.toml` | Path to
|
|
48
|
+
| `-c, --config <path>` | `./.dirsql.toml` | Path to a [config file](./config.md). **Repeatable** (`-c a -c b`): the configs load and merge in argv order — see [Composing multiple configs](./config.md#composing-multiple-configs). The index is always rooted at the **invocation directory** (the current working directory), regardless of where a config lives — so `--config /elsewhere/.dirsql.toml` still indexes the directory you ran `dirsql` from. With none given, `./.dirsql.toml` is used; when it does not exist, the server runs in [zero-config mode](#zero-config-mode). |
|
|
49
49
|
| `--host <addr>` | `localhost` | Bind address. |
|
|
50
50
|
| `--port <n>` | `7117` | TCP port to bind. |
|
|
51
51
|
| `--persist [<path>]` | off | Keep the SQLite index on disk between runs so a restart only re-parses files that actually changed. Bare `--persist` caches at `<root>/.dirsql/cache.db`; `--persist <path>` caches at `<path>`. Off by default (the index is ephemeral). Global — also honored by [`dirsql query`](#dirsql-query). See [Keep the index across restarts](../howto/persist.md). |
|
|
@@ -18,7 +18,7 @@ Declaring them requires nothing else. The names below aren't a protected or
|
|
|
18
18
|
enforced namespace — nothing stops you from declaring a column with one of
|
|
19
19
|
these names for an unrelated purpose, in which case dirsql's computed value
|
|
20
20
|
lands there like any other fact (unless your own row source — an `on-file`
|
|
21
|
-
command or SDK `
|
|
21
|
+
command or SDK `on_file` callback — supplies its own value for that name;
|
|
22
22
|
see [Precedence](#precedence)).
|
|
23
23
|
|
|
24
24
|
## Stat columns
|
|
@@ -71,8 +71,8 @@ A file at `_comments/abc123/2024-05-05.jsonl` produces a row with
|
|
|
71
71
|
## Precedence
|
|
72
72
|
|
|
73
73
|
Values produced by a table's own row source — an `on-file` command's JSON
|
|
74
|
-
output or an SDK `
|
|
75
|
-
auto-injected facts of the same name. An
|
|
74
|
+
output or an SDK `on_file` callback's return value — **win** over
|
|
75
|
+
auto-injected facts of the same name. An `on_file` callback that explicitly emits
|
|
76
76
|
`path` is honored.
|
|
77
77
|
|
|
78
78
|
Injection order per row: stat columns first, then glob captures, then
|
|
@@ -98,13 +98,13 @@ per-file command.
|
|
|
98
98
|
|---|---|---|
|
|
99
99
|
| `ddl` | yes | A SQLite `CREATE TABLE` statement. The table name is parsed from it. Only columns declared here are populated; auto-injected facts not in the DDL are dropped. |
|
|
100
100
|
| `glob` | yes | Glob pattern matched against root-relative paths. May contain `{name}` [capture segments](./columns.md#glob-captures). Every table whose glob matches a file receives that file's rows — a file can populate multiple tables. |
|
|
101
|
-
| `strict` | no (default `false`) | When `true`, rows whose keys do not exactly match the declared columns are rejected with an error: extra keys error, and every declared column must be supplied (by the command/
|
|
101
|
+
| `strict` | no (default `false`) | When `true`, rows whose keys do not exactly match the declared columns are rejected with an error: extra keys error, and every declared column must be supplied (by the command/on-file output, a glob capture, or a stat column). When `false`, extra keys are dropped and missing columns become `NULL`. |
|
|
102
102
|
| `on-file` | no | A command run once per matched file; its stdout (a JSON array of row objects) becomes the file's rows. Must be non-empty. See [Command hooks](./hooks.md#on-file). |
|
|
103
103
|
|
|
104
104
|
Without `on-file`, a table produces exactly one row per matched file, built
|
|
105
105
|
entirely from filesystem facts. Content interpretation (frontmatter, JSON
|
|
106
106
|
fields, CSV parsing) is out of scope for plain config tables — use
|
|
107
|
-
`on-file`, or a programmatic [SDK table](./sdk.md#table) with an `
|
|
107
|
+
`on-file`, or a programmatic [SDK table](./sdk.md#table) with an `on_file`
|
|
108
108
|
callback.
|
|
109
109
|
|
|
110
110
|
```toml
|
|
@@ -131,6 +131,38 @@ by the command wins over a same-named fact. Output that is not a JSON array
|
|
|
131
131
|
of objects is a per-file failure: the file is skipped with a stderr warning
|
|
132
132
|
and the scan continues (see [failure semantics](./hooks.md#failure-semantics)).
|
|
133
133
|
|
|
134
|
+
## Composing multiple configs
|
|
135
|
+
|
|
136
|
+
Pass [`-c`/`--config`](./cli.md#flags) more than once to compose several config
|
|
137
|
+
files — a shared team config plus local overrides, or a plugin's config
|
|
138
|
+
alongside your own:
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
dirsql -c ./.dirsql.toml -c ~/team/embeddings.toml -c ./local.toml
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
The configs load and merge in **argv order**:
|
|
145
|
+
|
|
146
|
+
- **`[[table]]`, `ignore`, and `[[dirsql.extension]]` entries accumulate** across
|
|
147
|
+
all configs, in order.
|
|
148
|
+
- **Each config's `on-file`, `pre-query`, and `post-query` hooks run from that
|
|
149
|
+
config file's own directory**, under that config's own
|
|
150
|
+
[`hook-timeout`](#dirsql-keys) — so a relative command like
|
|
151
|
+
`on-file = "sh ./extract.sh"` resolves against the config that declared it,
|
|
152
|
+
wherever it lives.
|
|
153
|
+
- **`pre-query` / `post-query` hooks chain FIFO**: the request body flows through
|
|
154
|
+
each `pre-query` stage in order to the final SQL, and the result rows flow
|
|
155
|
+
through each `post-query` stage to the response. See the
|
|
156
|
+
[hook contract](./hooks.md).
|
|
157
|
+
- Each config is **validated on its own** (the [parse errors](#parse-errors)
|
|
158
|
+
below apply per file). There is no cross-file merge validation, with one
|
|
159
|
+
structural exception: **two configs defining a table of the same name is an
|
|
160
|
+
error**, naming the table.
|
|
161
|
+
|
|
162
|
+
The index [root](./cli.md#flags) is the invocation directory regardless of where
|
|
163
|
+
any config lives. With no `-c`, `./.dirsql.toml` is used; a single `-c` behaves
|
|
164
|
+
exactly as before.
|
|
165
|
+
|
|
134
166
|
## Parse errors
|
|
135
167
|
|
|
136
168
|
Loading fails (the CLI enters [degraded mode](./cli.md#degraded-mode); the
|
|
@@ -296,19 +296,19 @@ rather than waiting — an intentional, language-idiomatic difference.
|
|
|
296
296
|
::: code-group
|
|
297
297
|
|
|
298
298
|
```python [Python]
|
|
299
|
-
Table(*, ddl: str, glob: str,
|
|
299
|
+
Table(*, ddl: str, glob: str, on_file: Callable[[str], list[dict]], strict: bool = False)
|
|
300
300
|
```
|
|
301
301
|
|
|
302
302
|
```typescript [TypeScript]
|
|
303
|
-
new Table({ ddl, glob,
|
|
303
|
+
new Table({ ddl, glob, onFile, strict? })
|
|
304
304
|
// or a plain object — TableDef and Table are interchangeable:
|
|
305
|
-
{ ddl: string, glob: string,
|
|
305
|
+
{ ddl: string, glob: string, onFile: (path: string) => Record<string, unknown>[], strict?: boolean }
|
|
306
306
|
```
|
|
307
307
|
|
|
308
308
|
```rust [Rust]
|
|
309
|
-
Table::new(ddl, glob,
|
|
310
|
-
Table::try_new(ddl, glob,
|
|
311
|
-
Table::strict(ddl, glob,
|
|
309
|
+
Table::new(ddl, glob, on_file) // on_file: Fn(&str) -> Vec<Row>, infallible
|
|
310
|
+
Table::try_new(ddl, glob, on_file) // on_file: Fn(&str) -> Result<Vec<Row>, _>
|
|
311
|
+
Table::strict(ddl, glob, on_file) // Table::new with strict = true
|
|
312
312
|
```
|
|
313
313
|
|
|
314
314
|
:::
|
|
@@ -321,7 +321,7 @@ Maps files to table rows.
|
|
|
321
321
|
`{name}` [captures](./columns.md#glob-captures). Every table whose glob
|
|
322
322
|
matches a file receives that file's rows — a file can populate multiple
|
|
323
323
|
tables.
|
|
324
|
-
- `
|
|
324
|
+
- `on_file` (`on_file` / `onFile`) — Callback receiving the matched file's full path (the root
|
|
325
325
|
joined with the file's relative path — absolute when `root` is absolute)
|
|
326
326
|
and returning the rows that file contributes. `dirsql` never reads file
|
|
327
327
|
contents itself; a callback that needs the body reads the path. Return
|
|
@@ -356,7 +356,7 @@ relative to the root. Errors never terminate the stream.
|
|
|
356
356
|
|
|
357
357
|
## Value types
|
|
358
358
|
|
|
359
|
-
An `
|
|
359
|
+
An `on_file` callback (and the SQLite columns it feeds) accepts:
|
|
360
360
|
|
|
361
361
|
| SQLite column | Python value | TypeScript value | Rust `Value` |
|
|
362
362
|
|---|---|---|---|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
//! PyO3 binding for `dirsql`. Intentionally thin: all orchestration lives in
|
|
2
2
|
//! `dirsql::DirSQL`. This layer only:
|
|
3
3
|
//!
|
|
4
|
-
//! - wraps a Python `
|
|
4
|
+
//! - wraps a Python `on_file` callable in a Rust closure (acquiring the GIL as
|
|
5
5
|
//! needed) so it can be handed to [`dirsql::Table`]
|
|
6
6
|
//! - converts row dicts, values, and events between Python and Rust
|
|
7
7
|
//! - forwards `new` / `query` / `_start_watcher` / `_poll_events` to the
|
|
@@ -21,7 +21,7 @@ mod python {
|
|
|
21
21
|
use std::time::Duration;
|
|
22
22
|
|
|
23
23
|
/// A table definition. Mirrors `dirsql::Table` but holds a Python
|
|
24
|
-
/// callable for `
|
|
24
|
+
/// callable for `on_file`.
|
|
25
25
|
#[pyclass(name = "Table", frozen)]
|
|
26
26
|
struct PyTable {
|
|
27
27
|
#[pyo3(get)]
|
|
@@ -29,7 +29,7 @@ mod python {
|
|
|
29
29
|
#[pyo3(get)]
|
|
30
30
|
glob: String,
|
|
31
31
|
#[pyo3(get)]
|
|
32
|
-
|
|
32
|
+
on_file: Py<PyAny>,
|
|
33
33
|
#[pyo3(get)]
|
|
34
34
|
strict: bool,
|
|
35
35
|
/// Parsed table name (from `ddl`) via `dirsql::db::parse_table_name`,
|
|
@@ -44,13 +44,13 @@ mod python {
|
|
|
44
44
|
#[pymethods]
|
|
45
45
|
impl PyTable {
|
|
46
46
|
#[new]
|
|
47
|
-
#[pyo3(signature = (*, ddl, glob,
|
|
48
|
-
fn new(ddl: String, glob: String,
|
|
47
|
+
#[pyo3(signature = (*, ddl, glob, on_file, strict=false))]
|
|
48
|
+
fn new(ddl: String, glob: String, on_file: Py<PyAny>, strict: bool) -> Self {
|
|
49
49
|
let name = parse_table_name(&ddl);
|
|
50
50
|
PyTable {
|
|
51
51
|
ddl,
|
|
52
52
|
glob,
|
|
53
|
-
|
|
53
|
+
on_file,
|
|
54
54
|
strict,
|
|
55
55
|
name,
|
|
56
56
|
}
|
|
@@ -195,11 +195,11 @@ mod python {
|
|
|
195
195
|
}
|
|
196
196
|
|
|
197
197
|
fn build_table(py: Python<'_>, t: &PyTable) -> Table {
|
|
198
|
-
let
|
|
198
|
+
let on_file_ref = t.on_file.clone_ref(py);
|
|
199
199
|
let mut table = Table::try_new(
|
|
200
200
|
t.ddl.clone(),
|
|
201
201
|
t.glob.clone(),
|
|
202
|
-
|
|
202
|
+
make_on_file_closure(on_file_ref),
|
|
203
203
|
);
|
|
204
204
|
table.strict = t.strict;
|
|
205
205
|
table
|
|
@@ -207,23 +207,23 @@ mod python {
|
|
|
207
207
|
|
|
208
208
|
type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
|
|
209
209
|
|
|
210
|
-
fn
|
|
211
|
-
|
|
210
|
+
fn make_on_file_closure(
|
|
211
|
+
on_file: Py<PyAny>,
|
|
212
212
|
) -> impl Fn(&str) -> std::result::Result<Vec<Row>, BoxError> + Send + Sync + 'static {
|
|
213
213
|
move |path: &str| {
|
|
214
214
|
Python::attach(|py| -> std::result::Result<Vec<Row>, BoxError> {
|
|
215
|
-
let result =
|
|
215
|
+
let result = on_file
|
|
216
216
|
.call1(py, (path,))
|
|
217
|
-
.map_err(|e| -> BoxError { Box::new(
|
|
217
|
+
.map_err(|e| -> BoxError { Box::new(OnFileError(e.to_string())) })?;
|
|
218
218
|
let raw: Vec<HashMap<String, Py<PyAny>>> = result
|
|
219
219
|
.extract(py)
|
|
220
|
-
.map_err(|e: PyErr| -> BoxError { Box::new(
|
|
220
|
+
.map_err(|e: PyErr| -> BoxError { Box::new(OnFileError(e.to_string())) })?;
|
|
221
221
|
|
|
222
222
|
let mut rows = Vec::with_capacity(raw.len());
|
|
223
223
|
for r in &raw {
|
|
224
224
|
rows.push(
|
|
225
225
|
convert_py_row(py, r)
|
|
226
|
-
.map_err(|e| -> BoxError { Box::new(
|
|
226
|
+
.map_err(|e| -> BoxError { Box::new(OnFileError(e.to_string())) })?,
|
|
227
227
|
);
|
|
228
228
|
}
|
|
229
229
|
Ok(rows)
|
|
@@ -232,13 +232,13 @@ mod python {
|
|
|
232
232
|
}
|
|
233
233
|
|
|
234
234
|
#[derive(Debug)]
|
|
235
|
-
struct
|
|
236
|
-
impl std::fmt::Display for
|
|
235
|
+
struct OnFileError(String);
|
|
236
|
+
impl std::fmt::Display for OnFileError {
|
|
237
237
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
238
238
|
f.write_str(&self.0)
|
|
239
239
|
}
|
|
240
240
|
}
|
|
241
|
-
impl std::error::Error for
|
|
241
|
+
impl std::error::Error for OnFileError {}
|
|
242
242
|
|
|
243
243
|
fn to_py_err<E: std::fmt::Display>(e: E) -> PyErr {
|
|
244
244
|
PyRuntimeError::new_err(e.to_string())
|
|
@@ -493,8 +493,8 @@ mod python {
|
|
|
493
493
|
}
|
|
494
494
|
|
|
495
495
|
#[test]
|
|
496
|
-
fn
|
|
497
|
-
assert_eq!(
|
|
496
|
+
fn on_file_error_displays_inner() {
|
|
497
|
+
assert_eq!(OnFileError("bad".to_string()).to_string(), "bad");
|
|
498
498
|
}
|
|
499
499
|
|
|
500
500
|
#[test]
|