dirsql 0.3.23__tar.gz → 0.3.25__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. {dirsql-0.3.23 → dirsql-0.3.25}/Cargo.lock +1 -1
  2. {dirsql-0.3.23 → dirsql-0.3.25}/PKG-INFO +1 -1
  3. {dirsql-0.3.23 → dirsql-0.3.25}/docs/api/index.md +6 -4
  4. {dirsql-0.3.23/packages/python → dirsql-0.3.25}/docs/cli/config.md +73 -32
  5. {dirsql-0.3.23/packages/rust → dirsql-0.3.25}/docs/cli/http-api.md +16 -4
  6. {dirsql-0.3.23 → dirsql-0.3.25}/docs/cli/index.md +2 -2
  7. {dirsql-0.3.23 → dirsql-0.3.25}/docs/cli/server.md +2 -2
  8. {dirsql-0.3.23 → dirsql-0.3.25}/docs/getting-started.md +1 -1
  9. {dirsql-0.3.23 → dirsql-0.3.25}/docs/guide/async.md +55 -12
  10. {dirsql-0.3.23/packages/python → dirsql-0.3.25}/docs/guide/crdt.md +1 -1
  11. {dirsql-0.3.23 → dirsql-0.3.25}/docs/guide/querying.md +3 -2
  12. {dirsql-0.3.23/packages/rust → dirsql-0.3.25}/docs/guide/tables.md +1 -1
  13. {dirsql-0.3.23 → dirsql-0.3.25}/docs/guide/watching.md +6 -2
  14. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/Cargo.toml +1 -1
  15. {dirsql-0.3.23/packages/rust → dirsql-0.3.25/packages/python}/docs/api/index.md +6 -4
  16. {dirsql-0.3.23 → dirsql-0.3.25/packages/python}/docs/cli/config.md +73 -32
  17. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/cli/http-api.md +16 -4
  18. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/cli/index.md +2 -2
  19. {dirsql-0.3.23/packages/rust → dirsql-0.3.25/packages/python}/docs/cli/server.md +2 -2
  20. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/getting-started.md +1 -1
  21. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/guide/async.md +55 -12
  22. {dirsql-0.3.23 → dirsql-0.3.25/packages/python}/docs/guide/crdt.md +1 -1
  23. {dirsql-0.3.23/packages/rust → dirsql-0.3.25/packages/python}/docs/guide/querying.md +3 -2
  24. {dirsql-0.3.23 → dirsql-0.3.25/packages/python}/docs/guide/tables.md +1 -1
  25. {dirsql-0.3.23/packages/rust → dirsql-0.3.25/packages/python}/docs/guide/watching.md +6 -2
  26. {dirsql-0.3.23/packages/python → dirsql-0.3.25/packages/rust}/docs/api/index.md +6 -4
  27. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/docs/cli/config.md +73 -32
  28. {dirsql-0.3.23 → dirsql-0.3.25/packages/rust}/docs/cli/http-api.md +16 -4
  29. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/docs/cli/index.md +2 -2
  30. {dirsql-0.3.23/packages/python → dirsql-0.3.25/packages/rust}/docs/cli/server.md +2 -2
  31. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/docs/getting-started.md +1 -1
  32. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/docs/guide/async.md +55 -12
  33. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/docs/guide/crdt.md +1 -1
  34. {dirsql-0.3.23/packages/python → dirsql-0.3.25/packages/rust}/docs/guide/querying.md +3 -2
  35. {dirsql-0.3.23/packages/python → dirsql-0.3.25/packages/rust}/docs/guide/tables.md +1 -1
  36. {dirsql-0.3.23/packages/python → dirsql-0.3.25/packages/rust}/docs/guide/watching.md +6 -2
  37. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/src/config.rs +0 -17
  38. dirsql-0.3.25/packages/rust/src/scanner.rs +83 -0
  39. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/src/watcher.rs +5 -124
  40. dirsql-0.3.25/packages/rust/tests/config.rs +26 -0
  41. dirsql-0.3.25/packages/rust/tests/scanner.rs +97 -0
  42. dirsql-0.3.25/packages/rust/tests/watcher.rs +139 -0
  43. dirsql-0.3.23/packages/rust/src/scanner.rs +0 -136
  44. {dirsql-0.3.23 → dirsql-0.3.25}/Cargo.toml +0 -0
  45. {dirsql-0.3.23 → dirsql-0.3.25}/README.md +0 -0
  46. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/__init__.py +0 -0
  47. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/_async.py +0 -0
  48. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/_async_test.py +0 -0
  49. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/_dirsql.pyi +0 -0
  50. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/cli/__init__.py +0 -0
  51. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/cli/binary_path.py +0 -0
  52. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/cli/binary_path_test.py +0 -0
  53. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/cli/interpret/__init__.py +0 -0
  54. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/cli/interpret/dispatch_extract.py +0 -0
  55. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/cli/interpret/dispatch_extract_test.py +0 -0
  56. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/cli/interpret/load_app.py +0 -0
  57. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/cli/interpret/load_app_test.py +0 -0
  58. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/cli/interpret/run.py +0 -0
  59. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/cli/interpret/run_test.py +0 -0
  60. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/cli/interpret/write_message.py +0 -0
  61. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/cli/interpret/write_message_test.py +0 -0
  62. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/cli/is_windows.py +0 -0
  63. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/cli/is_windows_test.py +0 -0
  64. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/cli/main.py +0 -0
  65. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/cli/main_test.py +0 -0
  66. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/py.typed +0 -0
  67. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/resolve_config.py +0 -0
  68. {dirsql-0.3.23 → dirsql-0.3.25}/dirsql/resolve_config_test.py +0 -0
  69. {dirsql-0.3.23 → dirsql-0.3.25}/docs/.claude/CLAUDE.md +0 -0
  70. {dirsql-0.3.23 → dirsql-0.3.25}/docs/.vitepress/config.ts +0 -0
  71. {dirsql-0.3.23 → dirsql-0.3.25}/docs/.vitepress/theme/index.ts +0 -0
  72. {dirsql-0.3.23 → dirsql-0.3.25}/docs/.vitepress/theme/lang.ts +0 -0
  73. {dirsql-0.3.23 → dirsql-0.3.25}/docs/AGENTS.md +0 -0
  74. {dirsql-0.3.23 → dirsql-0.3.25}/docs/cli/init.md +0 -0
  75. {dirsql-0.3.23 → dirsql-0.3.25}/docs/guide/persistence.md +0 -0
  76. {dirsql-0.3.23 → dirsql-0.3.25}/docs/index.md +0 -0
  77. {dirsql-0.3.23 → dirsql-0.3.25}/docs/migrations.md +0 -0
  78. {dirsql-0.3.23 → dirsql-0.3.25}/docs/package.json +0 -0
  79. {dirsql-0.3.23 → dirsql-0.3.25}/docs/playwright.config.ts +0 -0
  80. {dirsql-0.3.23 → dirsql-0.3.25}/docs/pnpm-lock.yaml +0 -0
  81. {dirsql-0.3.23 → dirsql-0.3.25}/docs/pnpm-workspace.yaml +0 -0
  82. {dirsql-0.3.23 → dirsql-0.3.25}/docs/tests/integration/home.spec.ts +0 -0
  83. {dirsql-0.3.23 → dirsql-0.3.25}/docs/tests/integration/language-flag.spec.ts +0 -0
  84. {dirsql-0.3.23 → dirsql-0.3.25}/docs/tests/unit/config.test.ts +0 -0
  85. {dirsql-0.3.23 → dirsql-0.3.25}/docs/tests/unit/lang.test.ts +0 -0
  86. {dirsql-0.3.23 → dirsql-0.3.25}/docs/vitest.config.ts +0 -0
  87. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/README.md +0 -0
  88. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/conftest.py +0 -0
  89. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/.claude/CLAUDE.md +0 -0
  90. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/.vitepress/config.ts +0 -0
  91. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/.vitepress/theme/index.ts +0 -0
  92. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/.vitepress/theme/lang.ts +0 -0
  93. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/AGENTS.md +0 -0
  94. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/cli/init.md +0 -0
  95. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/guide/persistence.md +0 -0
  96. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/index.md +0 -0
  97. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/migrations.md +0 -0
  98. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/package.json +0 -0
  99. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/playwright.config.ts +0 -0
  100. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/pnpm-lock.yaml +0 -0
  101. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/pnpm-workspace.yaml +0 -0
  102. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/tests/integration/home.spec.ts +0 -0
  103. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/tests/integration/language-flag.spec.ts +0 -0
  104. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/tests/unit/config.test.ts +0 -0
  105. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/tests/unit/lang.test.ts +0 -0
  106. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/docs/vitest.config.ts +0 -0
  107. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/src/lib.rs +0 -0
  108. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/__init__.py +0 -0
  109. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/conftest.py +0 -0
  110. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/e2e/__init__.py +0 -0
  111. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/__fixtures__/data/a/meta.json +0 -0
  112. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/__fixtures__/data/b/meta.json +0 -0
  113. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/__fixtures__/dirsql.config.py +0 -0
  114. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/__fixtures__/interpret/data/a/meta.json +0 -0
  115. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/__fixtures__/interpret/data/b/meta.json +0 -0
  116. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/__fixtures__/interpret/dirsql.config.py +0 -0
  117. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/__fixtures__/interpret/dirsql.config_no_app.py +0 -0
  118. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/__fixtures__/interpret/dirsql.config_raises.py +0 -0
  119. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/__init__.py +0 -0
  120. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/interpret_subprocess.py +0 -0
  121. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/test_async_dirsql.py +0 -0
  122. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/test_binding.py +0 -0
  123. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/test_dirsql.py +0 -0
  124. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/test_docs_examples.py +0 -0
  125. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/test_docs_gaps.py +0 -0
  126. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/test_from_config.py +0 -0
  127. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/test_interpret.py +0 -0
  128. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/test_native_config.py +0 -0
  129. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/test_persist.py +0 -0
  130. {dirsql-0.3.23 → dirsql-0.3.25}/packages/python/tests/integration/test_serialization.py +0 -0
  131. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/Cargo.toml +0 -0
  132. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/README.md +0 -0
  133. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/benches/db_bench.rs +0 -0
  134. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/benches/differ_bench.rs +0 -0
  135. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/benches/matcher_bench.rs +0 -0
  136. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/benches/scanner_bench.rs +0 -0
  137. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/docs/cli/init.md +0 -0
  138. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/docs/guide/persistence.md +0 -0
  139. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/docs/index.md +0 -0
  140. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/docs/migrations.md +0 -0
  141. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/src/bin/dirsql.rs +0 -0
  142. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/src/cli/init.rs +0 -0
  143. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/src/cli/mod.rs +0 -0
  144. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/src/cli/native_config.rs +0 -0
  145. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/src/cli/router.rs +0 -0
  146. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/src/cli/serialize.rs +0 -0
  147. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/src/cli/server.rs +0 -0
  148. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/src/db.rs +0 -0
  149. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/src/differ.rs +0 -0
  150. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/src/lib.rs +0 -0
  151. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/src/matcher.rs +0 -0
  152. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/src/persist.rs +0 -0
  153. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/tests/async_sdk.rs +0 -0
  154. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/tests/cli_e2e.rs +0 -0
  155. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/tests/cli_integration.rs +0 -0
  156. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/tests/code_review_findings.rs +0 -0
  157. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/tests/docs_examples.rs +0 -0
  158. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/tests/docs_gaps.rs +0 -0
  159. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/tests/extensions.rs +0 -0
  160. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/tests/from_config.rs +0 -0
  161. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/tests/init_e2e.rs +0 -0
  162. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/tests/init_integration.rs +0 -0
  163. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/tests/persist.rs +0 -0
  164. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/tests/readonly_query.rs +0 -0
  165. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/tests/sdk.rs +0 -0
  166. {dirsql-0.3.23 → dirsql-0.3.25}/packages/rust/tests/serialization.rs +0 -0
  167. {dirsql-0.3.23 → dirsql-0.3.25}/pyproject.toml +0 -0
@@ -499,7 +499,7 @@ dependencies = [
499
499
 
500
500
  [[package]]
501
501
  name = "dirsql-py-ext"
502
- version = "0.3.23"
502
+ version = "0.3.25"
503
503
  dependencies = [
504
504
  "dirsql",
505
505
  "pyo3",
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dirsql
3
- Version: 0.3.23
3
+ Version: 0.3.25
4
4
  Requires-Dist: pytest>=8 ; extra == 'dev'
5
5
  Requires-Dist: pytest-describe>=2 ; extra == 'dev'
6
6
  Requires-Dist: pytest-asyncio>=0.23 ; extra == 'dev'
@@ -188,11 +188,12 @@ import { Table } from 'dirsql';
188
188
  ::: code-group
189
189
 
190
190
  ```python [Python]
191
- Table(*, ddl: str, glob: str, extract: Callable[[str], list[dict]])
191
+ Table(*, ddl: str, glob: str, extract: Callable[[str], list[dict]], strict: bool = False)
192
192
  ```
193
193
 
194
194
  ```rust [Rust]
195
- Table::new(ddl: &str, glob: &str, extract: fn(&str) -> Vec<Value>)
195
+ // Row = HashMap<String, Value>
196
+ Table::new(ddl: &str, glob: &str, extract: fn(&str) -> Vec<HashMap<String, Value>>)
196
197
  ```
197
198
 
198
199
  ```typescript [TypeScript]
@@ -207,7 +208,8 @@ Defines a mapping from files to SQLite table rows.
207
208
 
208
209
  - `ddl` -- A `CREATE TABLE` statement. The table name is parsed from this DDL.
209
210
  - `glob` -- A glob pattern matched against file paths relative to the root directory.
210
- - `extract` -- A callable `(path) -> list[dict]`. Receives the absolute filesystem path of the matched file. `dirsql` does not read file contents; a callback that needs the file body reads `path` itself. Returns a list of dicts/maps mapping column names to values. Return an empty list to skip a file.
211
+ - `extract` -- A callable `(path) -> list[dict]`. Receives the path of the matched file -- relative to the scan root, or absolute when `root` is absolute. `dirsql` does not read file contents; a callback that needs the file body reads `path` itself. Returns a list of dicts/maps mapping column names to values. Return an empty list to skip a file.
212
+ - `strict` -- Optional (default `False`). Controls row/schema validation. In the default relaxed mode, extra row keys are dropped and missing columns become `NULL`. When `True`, every row key must be a valid column identifier and any extra or missing key raises an error. Surfaced in [serialization](#serialization) above as part of each table's `{ ddl, glob, strict }`.
211
213
 
212
214
  **Attributes:**
213
215
 
@@ -231,7 +233,7 @@ use dirsql::RowEvent;
231
233
  ```
232
234
 
233
235
  ```typescript [TypeScript]
234
- import { RowEvent } from 'dirsql';
236
+ import type { RowEvent } from 'dirsql';
235
237
  ```
236
238
 
237
239
  :::
@@ -6,12 +6,24 @@ canonical: https://thekevinscott.github.io/dirsql/cli/config
6
6
 
7
7
  > Online: <https://thekevinscott.github.io/dirsql/cli/config>
8
8
 
9
- `dirsql` can be configured with an optional config file (if omitted, server falls back to [defaults](./server.md#defaults)). Two formats are accepted:
9
+ `dirsql` is configured with an optional config file; with none, the server
10
+ falls back to [zero-config defaults](./server.md#defaults). Choose a format by
11
+ what you need:
10
12
 
11
- - **`dirsql.toml`** — declarative; covers filesystem-fact tables. Works with any installation.
12
- - **`.py` / `.js`** — native-language; lets you write `extract` callbacks in Python or JavaScript. CLI-only, and only the launcher matching the file's language can run it. See [Native-Language Configs](#native-language-configs).
13
+ - **[TOML](#toml)** — declarative; defines filesystem-fact tables (the path,
14
+ glob captures, and stat metadata). Works with any installation.
15
+ - **[Python](#python)** and **[JavaScript](#javascript)** — native-language
16
+ configs that build tables from the *contents* of files (frontmatter, JSON
17
+ values, CSV cells) through a dynamic `extract` callback. CLI-only; only the
18
+ launcher matching the file's language can run it.
13
19
 
14
- ## Basic Example
20
+ ## TOML
21
+
22
+ Reach for a TOML config — the default `.dirsql.toml` — to declare tables from
23
+ filesystem facts: a glob selects files, and columns come from path captures and
24
+ stat metadata. No code required, and it works with every installation.
25
+
26
+ ### Basic Example
15
27
 
16
28
  ```toml
17
29
  [dirsql]
@@ -24,9 +36,11 @@ glob = "posts/*.md"
24
36
 
25
37
  Each `posts/*.md` file produces one row in the `posts` table.
26
38
 
27
- ## Loading a Config File
39
+ ### Loading a Config File
28
40
 
29
- Pass the config file path to the `DirSQL` constructor:
41
+ The CLI loads `./.dirsql.toml` by default; pass `--config <path>` to point at
42
+ another file. To load the same `.toml` from the SDK, pass its path to the
43
+ `DirSQL` constructor:
30
44
 
31
45
  ::: code-group
32
46
 
@@ -60,7 +74,7 @@ directory. Override it by passing `root` explicitly (the explicit value
60
74
  wins and a warning is emitted) or by declaring `[dirsql].root` in the
61
75
  config file itself.
62
76
 
63
- ## Root Directory
77
+ ### Root Directory
64
78
 
65
79
  By default, the config file's parent directory is the scan root. To index
66
80
  a different location, declare `[dirsql].root` (relative paths are resolved
@@ -72,7 +86,7 @@ root = "../data"
72
86
  ignore = ["node_modules/**"]
73
87
  ```
74
88
 
75
- ## Stat Virtuals
89
+ ### Stat Virtuals
76
90
 
77
91
  Every config-defined table can expose any of these reserved columns. Add
78
92
  the ones you want to your DDL; the rest are silently dropped.
@@ -96,7 +110,7 @@ WHERE _mtime > strftime('%s', '2024-01-01')
96
110
  ORDER BY _mtime DESC;
97
111
  ```
98
112
 
99
- ## Path Captures
113
+ ### Path Captures
100
114
 
101
115
  Use `{name}` in glob patterns to extract path segments as columns. Add a
102
116
  matching column name to the DDL and the capture is auto-populated:
@@ -111,7 +125,7 @@ A file at `_comments/abc123/2024-05-05.jsonl` produces a row with
111
125
  `thread_id = "abc123"`, `_basename = "2024-05-05.jsonl"`, and `_mtime` set
112
126
  to the file's modification time.
113
127
 
114
- ## Ignore Patterns
128
+ ### Ignore Patterns
115
129
 
116
130
  The `ignore` list skips files and directories entirely (not even scanned):
117
131
 
@@ -124,7 +138,7 @@ The top-level `.dirsql/` directory is always excluded, whether you list it
124
138
  or not — it is a reserved namespace for `dirsql`'s own metadata (see
125
139
  [Persistence](../guide/persistence.md)).
126
140
 
127
- ## Persistence
141
+ ### Persistence
128
142
 
129
143
  Set `persist = true` to keep the SQLite database on disk between runs
130
144
  instead of rebuilding from scratch on every startup:
@@ -138,7 +152,7 @@ persist = true
138
152
  See [Persistence](../guide/persistence.md) for the full reconcile algorithm,
139
153
  storage layout, and limitations.
140
154
 
141
- ## Loading extensions
155
+ ### Loading extensions
142
156
 
143
157
  You can load SQLite extensions by specifying them in a config.
144
158
 
@@ -165,7 +179,7 @@ be declared as a `[[table]]` — `dirsql` tables are per-file row tables — so
165
179
  `CREATE VIRTUAL TABLE` DDL is rejected; call the extension's functions in your
166
180
  queries instead.
167
181
 
168
- ## Strict Mode
182
+ ### Strict Mode
169
183
 
170
184
  By default, auto-injected virtuals that aren't in the DDL are silently
171
185
  dropped, and undeclared user-extract keys are dropped. Enable strict mode
@@ -180,10 +194,11 @@ strict = true
180
194
 
181
195
  Strict mode does **not** apply to auto-injected stat virtuals — those are
182
196
  always filtered to the DDL's declared columns regardless. Strict mode
183
- applies only to keys produced by an extract callback (relevant for
184
- programmatic [tables](../guide/tables.md)).
197
+ applies only to keys produced by an extract callback (relevant for the
198
+ [Python](#python) / [JavaScript](#javascript) configs below and programmatic
199
+ [tables](../guide/tables.md)).
185
200
 
186
- ## Full Example
201
+ ### Full Example
187
202
 
188
203
  ```toml
189
204
  [dirsql]
@@ -202,20 +217,18 @@ ddl = "CREATE TABLE logs (_path TEXT, _size INTEGER, _mtime INTEGER)"
202
217
  glob = "logs/*.csv"
203
218
  ```
204
219
 
205
- ## Native-Language Configs
220
+ ## Python
206
221
 
207
- You can provide a config file in a particular language, allowing you to define a dynamic extract function. This can be useful for building a database based on the _contents_ of a file.
222
+ Reach for a Python config when your columns come from the *contents* of a
223
+ file — parsed JSON, frontmatter, CSV cells — rather than from filesystem
224
+ facts alone. You write a dynamic `extract` callback in Python, and the file
225
+ otherwise looks exactly like the in-process SDK construction (same `DirSQL` /
226
+ `Table` API):
208
227
 
209
228
  ```bash
210
229
  dirsql --config dirsql.config.py
211
- dirsql --config dirsql.config.js
212
230
  ```
213
231
 
214
- The file looks exactly like the in-process SDK construction — same
215
- `DirSQL` / `Table` API:
216
-
217
- ::: code-group
218
-
219
232
  ```python [dirsql.config.py]
220
233
  import json
221
234
  from dirsql import DirSQL, Table
@@ -224,8 +237,9 @@ def extract_meta(path):
224
237
  with open(path) as f:
225
238
  return [json.load(f)]
226
239
 
227
- # Python must export an `app` variable
240
+ # Python must export a module-level `app`.
228
241
  app = DirSQL(
242
+ root="papers", # required — see "Set a root" below
229
243
  tables=[
230
244
  Table(
231
245
  ddl="CREATE TABLE papers (title TEXT, _path TEXT)",
@@ -236,11 +250,26 @@ app = DirSQL(
236
250
  )
237
251
  ```
238
252
 
253
+ `extract` receives the path of each matched file and returns a list of rows
254
+ (one dict per row).
255
+
256
+ ## JavaScript
257
+
258
+ A JavaScript config gives you the same contents-driven `extract` in Node,
259
+ in either ES module or CommonJS form:
260
+
261
+ ```bash
262
+ dirsql --config dirsql.config.mjs
263
+ ```
264
+
265
+ ::: code-group
266
+
239
267
  ```javascript [dirsql.config.mjs]
240
268
  import { readFileSync } from "node:fs";
241
269
  import { DirSQL } from "dirsql";
242
270
 
243
271
  export default new DirSQL({
272
+ root: "papers", // required — see "Set a root" below
244
273
  tables: [
245
274
  {
246
275
  ddl: "CREATE TABLE papers (title TEXT, _path TEXT)",
@@ -256,6 +285,7 @@ const { readFileSync } = require("node:fs");
256
285
  const { DirSQL } = require("dirsql");
257
286
 
258
287
  module.exports = new DirSQL({
288
+ root: "papers", // required — see "Set a root" below
259
289
  tables: [
260
290
  {
261
291
  ddl: "CREATE TABLE papers (title TEXT, _path TEXT)",
@@ -268,10 +298,21 @@ module.exports = new DirSQL({
268
298
 
269
299
  :::
270
300
 
271
- Only the extension matters — the file can be named anything. `dirsql.config.{py,mjs,cjs}` is the suggested convention but not required.
272
-
273
- ### Module conventions
274
-
275
- - **Python (`.py`)** module-level `app = DirSQL(...)`.
276
- - **ESM (`.mjs`, or `.js` in an ESM package)** — `export default new DirSQL(...)`.
277
- - **CommonJS (`.cjs`, or `.js` in a CJS package)** — `module.exports = new DirSQL(...)`.
301
+ ## Notes for native-language configs
302
+
303
+ These apply to both the Python and JavaScript forms above.
304
+
305
+ - **Export the config.** Python exposes a module-level `app = DirSQL(...)`; an
306
+ ES module (`.mjs`, or `.js` in an ESM package) uses
307
+ `export default new DirSQL(...)`; CommonJS (`.cjs`, or `.js` in a CJS
308
+ package) uses `module.exports = new DirSQL(...)`. Only the extension
309
+ matters — the file can be named anything; `dirsql.config.{py,mjs,cjs}` is the
310
+ suggested convention, not a requirement.
311
+ - **Set a `root`.** Unlike TOML configs (which default the scan root to the
312
+ config file's directory), native-language configs require an explicit `root`.
313
+ Without one the Python launcher errors and the JavaScript launcher silently
314
+ indexes nothing.
315
+ - **Install the launcher on your `PATH`.** To run your `extract`, the server
316
+ spawns `dirsql interpret`, so the matching `dirsql` launcher must be installed
317
+ and on your `PATH` — a global `pip`/`uv` install for `.py`, or `npm` for
318
+ `.mjs` / `.cjs`. Only the launcher matching the file's language can run it.
@@ -33,12 +33,12 @@ On error, the server returns a non-2xx status with a JSON body:
33
33
  {"error": "syntax error near \"SLECT\""}
34
34
  ```
35
35
 
36
- Malformed SQL returns `400`, not `500` the client sent bad input. Missing / unreadable config returns `503`.
36
+ Malformed SQL returns `400`. An unreadable or malformed config returns `503`; a *missing* config is not an error the server serves the default `files` table.
37
37
 
38
38
  ```bash
39
39
  curl -s http://localhost:7117/query \
40
40
  -H 'content-type: application/json' \
41
- -d '{"sql":"SELECT COUNT(*) AS n FROM posts"}' \
41
+ -d '{"sql":"SELECT COUNT(*) AS n FROM files"}' \
42
42
  | jq
43
43
  ```
44
44
 
@@ -47,9 +47,14 @@ for SQL semantics, the read-only restriction, and the return format.
47
47
 
48
48
  ## `GET /events`
49
49
 
50
- Opens a [Server-Sent Events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events) stream of change events. Each `data:` payload is the same JSON schema the SDK emits from [`db.watch()`](../guide/watching.md#event-types):
50
+ Opens a [Server-Sent Events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events) stream of change events.
51
+
52
+ On stream open the server emits a single `ready` frame so clients have a reliable signal that the subscription is attached. Every subsequent frame is named `row`, and its `data:` payload is the same JSON schema the SDK emits from [`db.watch()`](../guide/watching.md#event-types):
51
53
 
52
54
  ```
55
+ event: ready
56
+ data: {}
57
+
53
58
  event: row
54
59
  data: {"action":"insert","table":"posts","file_path":"posts/hello.json","row":{"title":"Hello World","author":"alice"},"old_row":null}
55
60
 
@@ -60,7 +65,14 @@ event: row
60
65
  data: {"action":"delete","table":"posts","file_path":"posts/second.json","row":{"title":"Second Post","author":"bob"},"old_row":null}
61
66
  ```
62
67
 
63
- Errors during extraction appear as `{"action":"error",...}` events on the same stream. They do **not** terminate the stream a malformed file is a per-event problem, not a server-wide one.
68
+ Errors during extraction appear as `error` action events on the same `row` stream. An error event carries an `error` message string and does **not** include `row` or `old_row`:
69
+
70
+ ```
71
+ event: row
72
+ data: {"action":"error","table":"posts","file_path":"posts/broken.json","error":"Extract error: ..."}
73
+ ```
74
+
75
+ (`table` is `null` when the failure isn't tied to a specific table.) Errors do **not** terminate the stream — a malformed file is a per-event problem, not a server-wide one.
64
76
 
65
77
  ```bash
66
78
  curl -N http://localhost:7117/events
@@ -53,7 +53,7 @@ for the library-vs-CLI feature split.
53
53
 
54
54
  ## Quick start
55
55
 
56
- From a directory containing your files and a [`.dirsql.toml`](./config.md):
56
+ From any directory containing your files (no config needed):
57
57
 
58
58
  ```bash
59
59
  dirsql
@@ -66,7 +66,7 @@ Then query it over HTTP:
66
66
  ```bash
67
67
  curl -s http://localhost:7117/query \
68
68
  -H 'content-type: application/json' \
69
- -d '{"sql":"SELECT COUNT(*) AS n FROM posts"}' \
69
+ -d '{"sql":"SELECT COUNT(*) AS n FROM files"}' \
70
70
  | jq
71
71
  ```
72
72
 
@@ -28,7 +28,7 @@ $ Running at localhost:7117
28
28
 
29
29
  The server reads tables from a [config file](./config.md). By default it
30
30
  looks for `./.dirsql.toml`; pass `--config <path>` to point at a different
31
- `.toml` file or a [native-language config](./config.md#native-language-configs)
31
+ `.toml` file or a [native-language config](./config.md)
32
32
  (`.py` / `.js`).
33
33
 
34
34
  ## Defaults
@@ -53,7 +53,7 @@ A config file will override the default.
53
53
 
54
54
  | Flag | Default | Description |
55
55
  |---|---|---|
56
- | `--config <path>` | `./.dirsql.toml` | Path to the config file (`.toml` or [native-language](./config.md#native-language-configs)). The index is rooted at the directory containing this file. |
56
+ | `--config <path>` | `./.dirsql.toml` | Path to the config file (`.toml` or [native-language](./config.md)). The index is rooted at the directory containing this file. |
57
57
  | `--host <addr>` | `localhost` | Bind address |
58
58
  | `--port <n>` | `7117` | TCP port to bind |
59
59
 
@@ -171,7 +171,7 @@ const results = await db.query(`
171
171
 
172
172
  1. `dirsql` walks the directory tree
173
173
  2. Files matching each table's glob pattern are identified
174
- 3. The `extract` function receives each matched file's absolute path and returns rows
174
+ 3. The `extract` function receives each matched file's path (relative to the scan root, or absolute when `root` is absolute) and returns rows
175
175
  4. Rows are inserted into an in-memory SQLite database
176
176
  5. SQL queries run against that database
177
177
 
@@ -28,6 +28,7 @@ async def main():
28
28
  ),
29
29
  ],
30
30
  )
31
+ await db.ready()
31
32
 
32
33
  # Query (runs in a thread, does not block the event loop)
33
34
  results = await db.query("SELECT * FROM items WHERE value > 10")
@@ -37,7 +38,31 @@ asyncio.run(main())
37
38
  ```
38
39
 
39
40
  ```rust [Rust]
40
- use dirsql::{DirSQL, Table};
41
+ use dirsql::{DirSQL, Table, Value};
42
+ use std::collections::HashMap;
43
+
44
+ // See `row_from_json` in getting-started.md for a reusable helper that
45
+ // turns a JSON object into a dirsql row (dirsql::Value is not Deserialize,
46
+ // so a row can't be produced by serde_json::from_str directly).
47
+ fn row_from_json(raw: &str) -> HashMap<String, Value> {
48
+ let v: serde_json::Value = serde_json::from_str(raw).unwrap();
49
+ let serde_json::Value::Object(obj) = v else { return HashMap::new() };
50
+ obj.into_iter()
51
+ .map(|(k, val)| {
52
+ let v = match val {
53
+ serde_json::Value::String(s) => Value::Text(s),
54
+ serde_json::Value::Number(n) => n
55
+ .as_i64()
56
+ .map(Value::Integer)
57
+ .unwrap_or_else(|| Value::Real(n.as_f64().unwrap_or(0.0))),
58
+ serde_json::Value::Bool(b) => Value::Integer(b as i64),
59
+ serde_json::Value::Null => Value::Null,
60
+ other => Value::Text(other.to_string()),
61
+ };
62
+ (k, v)
63
+ })
64
+ .collect()
65
+ }
41
66
 
42
67
  #[tokio::main]
43
68
  async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -47,7 +72,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
47
72
  Table::new(
48
73
  "CREATE TABLE items (name TEXT, value INTEGER)",
49
74
  "data/*.json",
50
- |path| vec![serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap()],
75
+ |path| vec![row_from_json(&std::fs::read_to_string(path).unwrap())],
51
76
  ),
52
77
  ],
53
78
  )?;
@@ -83,7 +108,7 @@ console.log(results);
83
108
  ## Constructor
84
109
 
85
110
  ```python
86
- DirSQL(root=None, *, tables=None, ignore=None, config=None)
111
+ DirSQL(root=None, *, tables=None, ignore=None, config=None, persist=False, persist_path=None)
87
112
  ```
88
113
 
89
114
  The constructor immediately starts scanning in a background thread via `asyncio.ensure_future`. The constructor itself returns immediately without blocking.
@@ -132,15 +157,29 @@ async for event in db.watch():
132
157
  ```
133
158
 
134
159
  ```rust [Rust]
160
+ // `RowEvent` is an enum; match on the variant to destructure its fields.
161
+ // `StreamExt` (for `.next()`) comes from the `futures` crate, which is only a
162
+ // dirsql dependency under its `cli` feature -- add it to your own project:
163
+ //
164
+ // cargo add futures
165
+ use dirsql::RowEvent;
135
166
  use futures::StreamExt;
136
167
 
137
- let mut stream = db.watch();
168
+ let mut stream = db.watch()?; // watch() returns Result<WatchStream>
138
169
  while let Some(event) = stream.next().await {
139
- match event.action {
140
- Action::Insert => println!("New row in {}: {:?}", event.table, event.row),
141
- Action::Update => println!("Updated row in {}: {:?}", event.table, event.row),
142
- Action::Delete => println!("Deleted row from {}: {:?}", event.table, event.row),
143
- Action::Error => eprintln!("Error: {:?}", event.error),
170
+ match event {
171
+ RowEvent::Insert { table, row, file_path } => {
172
+ println!("New row in {table} ({file_path}): {row:?}")
173
+ }
174
+ RowEvent::Update { table, new_row, file_path, .. } => {
175
+ println!("Updated row in {table} ({file_path}): {new_row:?}")
176
+ }
177
+ RowEvent::Delete { table, row, file_path } => {
178
+ println!("Deleted row from {table} ({file_path}): {row:?}")
179
+ }
180
+ RowEvent::Error { file_path, error, .. } => {
181
+ eprintln!("Error on {file_path:?}: {error}")
182
+ }
144
183
  }
145
184
  }
146
185
  ```
@@ -188,16 +227,20 @@ async def main():
188
227
  ```
189
228
 
190
229
  ```rust [Rust]
191
- async fn watch_and_serve(db: &DirSQL) {
192
- let mut stream = db.watch();
230
+ // `.next()` needs `StreamExt` from the `futures` crate (`cargo add futures`).
231
+ use futures::StreamExt;
232
+
233
+ async fn watch_and_serve(db: &DirSQL) -> Result<(), Box<dyn std::error::Error>> {
234
+ let mut stream = db.watch()?; // watch() returns Result<WatchStream>
193
235
  while let Some(event) = stream.next().await {
194
236
  notify_clients(&event).await;
195
237
  }
238
+ Ok(())
196
239
  }
197
240
 
198
241
  #[tokio::main]
199
242
  async fn main() -> Result<(), Box<dyn std::error::Error>> {
200
- let db = DirSQL::new("./data", vec![...])?;
243
+ let db = DirSQL::new("./data", vec![/* tables */])?;
201
244
 
202
245
  tokio::join!(
203
246
  watch_and_serve(&db),
@@ -94,7 +94,7 @@ const tables: TableDef[] = [
94
94
  },
95
95
  ];
96
96
 
97
- const db = new DirSQL('./workspace', tables);
97
+ const db = new DirSQL({ root: './workspace', tables });
98
98
  ```
99
99
 
100
100
  :::
@@ -160,8 +160,9 @@ assert!(matches!(err, dirsql::DirSqlError::WriteForbidden));
160
160
  ```
161
161
 
162
162
  ```typescript [TypeScript]
163
- // Throws an Error whose message explains writes are not accepted.
164
- expect(() => db.query('DELETE FROM posts')).toThrow(/read-only/i);
163
+ // Rejects with an Error whose message explains writes are not accepted.
164
+ // `db.query` is async, so assert on the rejected promise.
165
+ await expect(db.query('DELETE FROM posts')).rejects.toThrow(/read-only/i);
165
166
  ```
166
167
 
167
168
  :::
@@ -91,7 +91,7 @@ Glob syntax follows standard Unix globbing rules. `**` matches any number of dir
91
91
 
92
92
  A callable `(path: str) -> list[dict]` that converts a file into rows.
93
93
 
94
- - `path` is the **absolute filesystem path** of the matched file
94
+ - `path` is the path of the matched file, **relative to the scan root** (or absolute when the `root` passed to `DirSQL` is absolute)
95
95
  - Return a list of dicts, where each dict maps column names to values
96
96
  - Return an empty list to skip a file
97
97
 
@@ -36,6 +36,10 @@ async for event in db.watch():
36
36
  ```
37
37
 
38
38
  ```rust [Rust]
39
+ // `StreamExt` (for `.next()`) comes from the `futures` crate. dirsql only
40
+ // depends on `futures` under its `cli` feature, so add it to your project:
41
+ //
42
+ // cargo add futures
39
43
  use dirsql::{DirSQL, RowEvent, Table, Value};
40
44
  use futures::StreamExt;
41
45
  use std::collections::HashMap;
@@ -84,8 +88,8 @@ while let Some(event) = stream.next().await {
84
88
  RowEvent::Delete { table, row, file_path } => {
85
89
  println!("delete on {table} ({file_path}): {row:?}")
86
90
  }
87
- RowEvent::Error { file_path, error } => {
88
- println!("error on {file_path:?}: {error}")
91
+ RowEvent::Error { table, file_path, error } => {
92
+ println!("error on {table:?} {file_path:?}: {error}")
89
93
  }
90
94
  }
91
95
  }
@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
4
4
  # pypi/maturin handler can rewrite it via `write-version` before
5
5
  # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
6
6
  # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
7
- version = "0.3.23"
7
+ version = "0.3.25"
8
8
  edition.workspace = true
9
9
  publish = false
10
10
  readme = "README.md"
@@ -188,11 +188,12 @@ import { Table } from 'dirsql';
188
188
  ::: code-group
189
189
 
190
190
  ```python [Python]
191
- Table(*, ddl: str, glob: str, extract: Callable[[str], list[dict]])
191
+ Table(*, ddl: str, glob: str, extract: Callable[[str], list[dict]], strict: bool = False)
192
192
  ```
193
193
 
194
194
  ```rust [Rust]
195
- Table::new(ddl: &str, glob: &str, extract: fn(&str) -> Vec<Value>)
195
+ // Row = HashMap<String, Value>
196
+ Table::new(ddl: &str, glob: &str, extract: fn(&str) -> Vec<HashMap<String, Value>>)
196
197
  ```
197
198
 
198
199
  ```typescript [TypeScript]
@@ -207,7 +208,8 @@ Defines a mapping from files to SQLite table rows.
207
208
 
208
209
  - `ddl` -- A `CREATE TABLE` statement. The table name is parsed from this DDL.
209
210
  - `glob` -- A glob pattern matched against file paths relative to the root directory.
210
- - `extract` -- A callable `(path) -> list[dict]`. Receives the absolute filesystem path of the matched file. `dirsql` does not read file contents; a callback that needs the file body reads `path` itself. Returns a list of dicts/maps mapping column names to values. Return an empty list to skip a file.
211
+ - `extract` -- A callable `(path) -> list[dict]`. Receives the path of the matched file -- relative to the scan root, or absolute when `root` is absolute. `dirsql` does not read file contents; a callback that needs the file body reads `path` itself. Returns a list of dicts/maps mapping column names to values. Return an empty list to skip a file.
212
+ - `strict` -- Optional (default `False`). Controls row/schema validation. In the default relaxed mode, extra row keys are dropped and missing columns become `NULL`. When `True`, every row key must be a valid column identifier and any extra or missing key raises an error. Surfaced in [serialization](#serialization) above as part of each table's `{ ddl, glob, strict }`.
211
213
 
212
214
  **Attributes:**
213
215
 
@@ -231,7 +233,7 @@ use dirsql::RowEvent;
231
233
  ```
232
234
 
233
235
  ```typescript [TypeScript]
234
- import { RowEvent } from 'dirsql';
236
+ import type { RowEvent } from 'dirsql';
235
237
  ```
236
238
 
237
239
  :::