dirsql 0.3.0__tar.gz → 0.3.1__tar.gz

{dirsql-0.3.0 → dirsql-0.3.1}/Cargo.lock

@@ -433,27 +433,6 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
 
-[[package]]
-name = "csv"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938"
-dependencies = [
- "csv-core",
- "itoa",
- "ryu",
- "serde_core",
-]
-
-[[package]]
-name = "csv-core"
-version = "0.1.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "ctor"
 version = "0.10.0"
@@ -485,7 +464,6 @@ dependencies = [
  "blake3",
  "clap",
  "criterion",
- "csv",
  "eventsource-client",
  "futures",
  "futures-channel",
@@ -500,7 +478,6 @@ dependencies = [
  "rusqlite",
  "serde",
  "serde_json",
- "serde_yaml",
  "tempfile",
  "thiserror",
  "tokio",
@@ -522,7 +499,7 @@ dependencies = [
 
 [[package]]
 name = "dirsql-py-ext"
-version = "0.3.0"
+version = "0.3.1"
 dependencies = [
  "dirsql",
  "pyo3",
@@ -2230,19 +2207,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "serde_yaml"
-version = "0.9.34+deprecated"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
-dependencies = [
- "indexmap",
- "itoa",
- "ryu",
- "serde",
- "unsafe-libyaml",
-]
-
 [[package]]
 name = "shlex"
 version = "1.3.0"
@@ -2633,12 +2597,6 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
 
-[[package]]
-name = "unsafe-libyaml"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
-
 [[package]]
 name = "untrusted"
 version = "0.9.0"

{dirsql-0.3.0 → dirsql-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql
-Version: 0.3.0
+Version: 0.3.1
 Requires-Dist: pytest>=8 ; extra == 'dev'
 Requires-Dist: pytest-describe>=2 ; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.23 ; extra == 'dev'

dirsql-0.3.1/docs/guide/config.md

@@ -0,0 +1,193 @@
+---
+canonical: https://thekevinscott.github.io/dirsql/guide/config
+---
+
+# Configuration File
+
+> Online: <https://thekevinscott.github.io/dirsql/guide/config>
+
+`dirsql` can be configured with a `.dirsql.toml` file. Tables defined this
+way produce **one row per matched file**. Each row's columns come from
+filesystem facts:
+
+- **Glob path captures** — named `{placeholder}` segments in the glob.
+- **Stat virtuals** — reserved `_`-prefixed columns for path-derived and
+  stat-derived metadata.
+
+Content interpretation (parsing JSON, CSV, frontmatter, etc.) is **not**
+configured in `.dirsql.toml`. If you need columns derived from file
+contents, register a programmatic [`Table`](./tables.md) whose `extract`
+function does the parsing in your host language.
+
+## Basic Example
+
+```toml
+[dirsql]
+ignore = ["node_modules/**", ".git/**"]
+
+[[table]]
+ddl  = "CREATE TABLE posts (_path TEXT, _basename TEXT, _size INTEGER, _mtime INTEGER)"
+glob = "posts/*.md"
+```
+
+Each `posts/*.md` file produces one row. The DDL declares which stat
+virtuals are surfaced as SQL columns.
+
+## Loading a Config File
+
+Pass the config file path to the `DirSQL` constructor:
+
+::: code-group
+
+```python [Python]
+from dirsql import DirSQL
+
+db = DirSQL(config="./my-project/.dirsql.toml")
+await db.ready()
+```
+
+```rust [Rust]
+use dirsql::DirSQL;
+
+let db = DirSQL::builder()
+    .config("./my-project/.dirsql.toml")
+    .build()?;
+```
+
+```typescript [TypeScript]
+import { DirSQL } from "dirsql";
+
+// String argument is interpreted as a config file path.
+const db = new DirSQL("./my-project/.dirsql.toml");
+await db.ready;
+```
+
+:::
+
+By default, the root directory scanned is the config file's parent
+directory. Override it by passing `root` explicitly (the explicit value
+wins and a warning is emitted) or by declaring `[dirsql].root` in the
+config file itself.
+
+## Root Directory
+
+By default, the config file's parent directory is the scan root. To index
+a different location, declare `[dirsql].root` (relative paths are resolved
+relative to the config file's parent):
+
+```toml
+[dirsql]
+root = "../data"
+ignore = ["node_modules/**"]
+```
+
+## Stat Virtuals
+
+Every config-defined table can expose any of these reserved columns. Add
+the ones you want to your DDL; the rest are silently dropped.
+
+| Column | Type    | Source |
+|--------|---------|--------|
+| `_path`     | TEXT    | The file's path relative to the scan root. |
+| `_basename` | TEXT    | The filename including extension. |
+| `_dir`      | TEXT    | The parent directory path (relative to root). |
+| `_ext`      | TEXT    | The file extension, lowercased, no leading dot. |
+| `_size`     | INTEGER | Size in bytes. |
+| `_mtime`    | INTEGER | Last-modified time, unix seconds. |
+| `_ctime`    | INTEGER | Created/changed time, unix seconds. |
+
+Example query:
+
+```sql
+SELECT _basename, _size
+FROM posts
+WHERE _mtime > strftime('%s', '2024-01-01')
+ORDER BY _mtime DESC;
+```
+
+## Path Captures
+
+Use `{name}` in glob patterns to extract path segments as columns. Add a
+matching column name to the DDL and the capture is auto-populated:
+
+```toml
+[[table]]
+ddl  = "CREATE TABLE comments (thread_id TEXT, _basename TEXT, _mtime INTEGER)"
+glob = "_comments/{thread_id}/*.jsonl"
+```
+
+A file at `_comments/abc123/2024-05-05.jsonl` produces a row with
+`thread_id = "abc123"`, `_basename = "2024-05-05.jsonl"`, and `_mtime` set
+to the file's modification time.
+
+## Ignore Patterns
+
+The `ignore` list skips files and directories entirely (not even scanned):
+
+```toml
+[dirsql]
+ignore = ["node_modules/**", ".git/**", "*.pyc", "__pycache__/**"]
+```
+
+The top-level `.dirsql/` directory is always excluded, whether you list it
+or not — it is a reserved namespace for `dirsql`'s own metadata (see
+[Persistence](./persistence.md)).
+
+## Persistence
+
+Set `persist = true` to keep the SQLite database on disk between runs
+instead of rebuilding from scratch on every startup:
+
+```toml
+[dirsql]
+persist = true
+# persist_path = ".dirsql/cache.db"   # optional; this is the default
+```
+
+See [Persistence](./persistence.md) for the full reconcile algorithm,
+storage layout, and limitations.
+
+## Strict Mode
+
+By default, auto-injected virtuals that aren't in the DDL are silently
+dropped, and undeclared user-extract keys are dropped. Enable strict mode
+to error when an extract emits keys not declared in the DDL:
+
+```toml
+[[table]]
+ddl  = "CREATE TABLE comments (thread_id TEXT)"
+glob = "_comments/{thread_id}/*.jsonl"
+strict = true
+```
+
+Strict mode does **not** apply to auto-injected stat virtuals — those are
+always filtered to the DDL's declared columns regardless. Strict mode
+applies only to keys produced by an extract callback (relevant for
+programmatic [tables](./tables.md)).
+
+## Full Example
+
+```toml
+[dirsql]
+ignore = ["node_modules/**", ".git/**", "dist/**"]
+
+[[table]]
+ddl  = "CREATE TABLE comments (thread_id TEXT, _basename TEXT, _mtime INTEGER)"
+glob = "_comments/{thread_id}/*.jsonl"
+
+[[table]]
+ddl  = "CREATE TABLE documents (_path TEXT, _basename TEXT, _size INTEGER)"
+glob = "**/index.md"
+
+[[table]]
+ddl  = "CREATE TABLE logs (_path TEXT, _size INTEGER, _mtime INTEGER)"
+glob = "logs/*.csv"
+```
+
+## When you need parsed content
+
+`.dirsql.toml` does not parse file contents. For columns derived from the
+*inside* of files (frontmatter keys, JSON values, CSV cells, etc.),
+register a programmatic [`Table`](./tables.md) instead, and parse the
+bytes in your host language. Glob captures and stat virtuals are still
+auto-injected into rows produced by your extract.

{dirsql-0.3.0 → dirsql-0.3.1}/packages/python/Cargo.toml

@@ -4,7 +4,7 @@ name = "dirsql-py-ext"
 # pypi/maturin handler can rewrite it via `write-version` before
 # `maturin build`. `pyproject.toml` declares `dynamic = ["version"]`
 # and maturin reads this field. Mirrors `packages/rust/Cargo.toml`.
-version = "0.3.0"
+version = "0.3.1"
 edition.workspace = true
 publish = false
 readme = "README.md"

dirsql-0.3.1/packages/python/docs/guide/config.md

@@ -0,0 +1,193 @@
+---
+canonical: https://thekevinscott.github.io/dirsql/guide/config
+---
+
+# Configuration File
+
+> Online: <https://thekevinscott.github.io/dirsql/guide/config>
+
+`dirsql` can be configured with a `.dirsql.toml` file. Tables defined this
+way produce **one row per matched file**. Each row's columns come from
+filesystem facts:
+
+- **Glob path captures** — named `{placeholder}` segments in the glob.
+- **Stat virtuals** — reserved `_`-prefixed columns for path-derived and
+  stat-derived metadata.
+
+Content interpretation (parsing JSON, CSV, frontmatter, etc.) is **not**
+configured in `.dirsql.toml`. If you need columns derived from file
+contents, register a programmatic [`Table`](./tables.md) whose `extract`
+function does the parsing in your host language.
+
+## Basic Example
+
+```toml
+[dirsql]
+ignore = ["node_modules/**", ".git/**"]
+
+[[table]]
+ddl  = "CREATE TABLE posts (_path TEXT, _basename TEXT, _size INTEGER, _mtime INTEGER)"
+glob = "posts/*.md"
+```
+
+Each `posts/*.md` file produces one row. The DDL declares which stat
+virtuals are surfaced as SQL columns.
+
+## Loading a Config File
+
+Pass the config file path to the `DirSQL` constructor:
+
+::: code-group
+
+```python [Python]
+from dirsql import DirSQL
+
+db = DirSQL(config="./my-project/.dirsql.toml")
+await db.ready()
+```
+
+```rust [Rust]
+use dirsql::DirSQL;
+
+let db = DirSQL::builder()
+    .config("./my-project/.dirsql.toml")
+    .build()?;
+```
+
+```typescript [TypeScript]
+import { DirSQL } from "dirsql";
+
+// String argument is interpreted as a config file path.
+const db = new DirSQL("./my-project/.dirsql.toml");
+await db.ready;
+```
+
+:::
+
+By default, the root directory scanned is the config file's parent
+directory. Override it by passing `root` explicitly (the explicit value
+wins and a warning is emitted) or by declaring `[dirsql].root` in the
+config file itself.
+
+## Root Directory
+
+By default, the config file's parent directory is the scan root. To index
+a different location, declare `[dirsql].root` (relative paths are resolved
+relative to the config file's parent):
+
+```toml
+[dirsql]
+root = "../data"
+ignore = ["node_modules/**"]
+```
+
+## Stat Virtuals
+
+Every config-defined table can expose any of these reserved columns. Add
+the ones you want to your DDL; the rest are silently dropped.
+
+| Column | Type    | Source |
+|--------|---------|--------|
+| `_path`     | TEXT    | The file's path relative to the scan root. |
+| `_basename` | TEXT    | The filename including extension. |
+| `_dir`      | TEXT    | The parent directory path (relative to root). |
+| `_ext`      | TEXT    | The file extension, lowercased, no leading dot. |
+| `_size`     | INTEGER | Size in bytes. |
+| `_mtime`    | INTEGER | Last-modified time, unix seconds. |
+| `_ctime`    | INTEGER | Created/changed time, unix seconds. |
+
+Example query:
+
+```sql
+SELECT _basename, _size
+FROM posts
+WHERE _mtime > strftime('%s', '2024-01-01')
+ORDER BY _mtime DESC;
+```
+
+## Path Captures
+
+Use `{name}` in glob patterns to extract path segments as columns. Add a
+matching column name to the DDL and the capture is auto-populated:
+
+```toml
+[[table]]
+ddl  = "CREATE TABLE comments (thread_id TEXT, _basename TEXT, _mtime INTEGER)"
+glob = "_comments/{thread_id}/*.jsonl"
+```
+
+A file at `_comments/abc123/2024-05-05.jsonl` produces a row with
+`thread_id = "abc123"`, `_basename = "2024-05-05.jsonl"`, and `_mtime` set
+to the file's modification time.
+
+## Ignore Patterns
+
+The `ignore` list skips files and directories entirely (not even scanned):
+
+```toml
+[dirsql]
+ignore = ["node_modules/**", ".git/**", "*.pyc", "__pycache__/**"]
+```
+
+The top-level `.dirsql/` directory is always excluded, whether you list it
+or not — it is a reserved namespace for `dirsql`'s own metadata (see
+[Persistence](./persistence.md)).
+
+## Persistence
+
+Set `persist = true` to keep the SQLite database on disk between runs
+instead of rebuilding from scratch on every startup:
+
+```toml
+[dirsql]
+persist = true
+# persist_path = ".dirsql/cache.db"   # optional; this is the default
+```
+
+See [Persistence](./persistence.md) for the full reconcile algorithm,
+storage layout, and limitations.
+
+## Strict Mode
+
+By default, auto-injected virtuals that aren't in the DDL are silently
+dropped, and undeclared user-extract keys are dropped. Enable strict mode
+to error when an extract emits keys not declared in the DDL:
+
+```toml
+[[table]]
+ddl  = "CREATE TABLE comments (thread_id TEXT)"
+glob = "_comments/{thread_id}/*.jsonl"
+strict = true
+```
+
+Strict mode does **not** apply to auto-injected stat virtuals — those are
+always filtered to the DDL's declared columns regardless. Strict mode
+applies only to keys produced by an extract callback (relevant for
+programmatic [tables](./tables.md)).
+
+## Full Example
+
+```toml
+[dirsql]
+ignore = ["node_modules/**", ".git/**", "dist/**"]
+
+[[table]]
+ddl  = "CREATE TABLE comments (thread_id TEXT, _basename TEXT, _mtime INTEGER)"
+glob = "_comments/{thread_id}/*.jsonl"
+
+[[table]]
+ddl  = "CREATE TABLE documents (_path TEXT, _basename TEXT, _size INTEGER)"
+glob = "**/index.md"
+
+[[table]]
+ddl  = "CREATE TABLE logs (_path TEXT, _size INTEGER, _mtime INTEGER)"
+glob = "logs/*.csv"
+```
+
+## When you need parsed content
+
+`.dirsql.toml` does not parse file contents. For columns derived from the
+*inside* of files (frontmatter keys, JSON values, CSV cells, etc.),
+register a programmatic [`Table`](./tables.md) instead, and parse the
+bytes in your host language. Glob captures and stat virtuals are still
+auto-injected into rows produced by your extract.

{dirsql-0.3.0 → dirsql-0.3.1}/packages/python/tests/integration/test_docs_gaps.py

@@ -1,8 +1,6 @@
 """Gap-filling tests for features documented in docs/ but previously untested.
 
 Each test cites the canonical doc location (docs page + section) that it covers.
-These were identified by the TESTS_AUDIT.md pass for bead dirsql-9ng
-(Tests follow docs: 1:1 mapping between documented features and tests).
 """
 
 import json
@@ -63,167 +61,48 @@ def describe_tables_guide_bytes_to_blob():
 
 
 # ---------------------------------------------------------------------------
-# docs/guide/config.md -- "Supported Formats" (.tsv/.ndjson/.toml/.yaml/.yml/.md)
-# and "Strict Mode" (strict = true)
+# docs/guide/tables.md -- "Strict Mode" (programmatic Table strict=True)
 # ---------------------------------------------------------------------------
 
 
-def describe_from_config_formats_gap():
+def describe_strict_mode_gap():
     @pytest.mark.asyncio
-    async def it_loads_tsv_files_via_config(config_dir):
-        """Docs (guide/config.md "Supported Formats"): .tsv format is tab-separated."""
-        _write(
-            os.path.join(config_dir, "data.tsv"),
-            "name\tcount\napples\t10\noranges\t20\n",
-        )
-        _write(
-            os.path.join(config_dir, ".dirsql.toml"),
-            """\
-[[table]]
-ddl = "CREATE TABLE produce (name TEXT, count TEXT)"
-glob = "*.tsv"
-""",
-        )
-        db = DirSQL(config=os.path.join(config_dir, ".dirsql.toml"))
-        await db.ready()
-        results = await db.query("SELECT * FROM produce ORDER BY name")
-        assert len(results) == 2
-        assert results[0]["name"] == "apples"
-        assert results[0]["count"] == "10"
-        assert results[1]["name"] == "oranges"
-
-    @pytest.mark.asyncio
-    async def it_loads_ndjson_files_via_config(config_dir):
-        """Docs (guide/config.md "Supported Formats"): .ndjson aliases JSONL (one row per line)."""
-        _write(
-            os.path.join(config_dir, "events.ndjson"),
-            json.dumps({"type": "click", "count": 5})
-            + "\n"
-            + json.dumps({"type": "view", "count": 100})
-            + "\n",
-        )
-        _write(
-            os.path.join(config_dir, ".dirsql.toml"),
-            """\
-[[table]]
-ddl = "CREATE TABLE events (type TEXT, count INTEGER)"
-glob = "*.ndjson"
-""",
-        )
-        db = DirSQL(config=os.path.join(config_dir, ".dirsql.toml"))
-        await db.ready()
-        results = await db.query("SELECT * FROM events ORDER BY type")
-        assert len(results) == 2
-        assert results[0]["type"] == "click"
-        assert results[0]["count"] == 5
-
-    @pytest.mark.asyncio
-    async def it_loads_toml_files_via_config(config_dir):
-        """Docs (guide/config.md "Supported Formats"): .toml format is one row per file."""
-        _write(
-            os.path.join(config_dir, "config", "app.toml"),
-            'name = "myapp"\nversion = "1.2"\n',
-        )
-        _write(
-            os.path.join(config_dir, ".dirsql.toml"),
-            """\
-[[table]]
-ddl = "CREATE TABLE app (name TEXT, version TEXT)"
-glob = "config/*.toml"
-""",
-        )
-        db = DirSQL(config=os.path.join(config_dir, ".dirsql.toml"))
-        await db.ready()
-        results = await db.query("SELECT * FROM app")
-        assert len(results) == 1
-        assert results[0]["name"] == "myapp"
-        assert results[0]["version"] == "1.2"
-
-    @pytest.mark.asyncio
-    @pytest.mark.parametrize("ext", ["yaml", "yml"])
-    async def it_loads_yaml_files_via_config(config_dir, ext):
-        """Docs (guide/config.md "Supported Formats"): .yaml/.yml mapping = 1 row."""
-        _write(
-            os.path.join(config_dir, f"data.{ext}"),
-            "name: widget\nprice: 9.99\n",
-        )
-        _write(
-            os.path.join(config_dir, ".dirsql.toml"),
-            f"""\
-[[table]]
-ddl = "CREATE TABLE items (name TEXT, price REAL)"
-glob = "*.{ext}"
-""",
-        )
-        db = DirSQL(config=os.path.join(config_dir, ".dirsql.toml"))
-        await db.ready()
-        results = await db.query("SELECT * FROM items")
-        assert len(results) == 1
-        assert results[0]["name"] == "widget"
-        assert results[0]["price"] == pytest.approx(9.99)
-
-    @pytest.mark.asyncio
-    async def it_loads_markdown_with_frontmatter_via_config(config_dir):
-        """Docs (guide/config.md "Supported Formats"): .md uses YAML frontmatter + body column."""
-        _write(
-            os.path.join(config_dir, "posts", "hello.md"),
-            "---\ntitle: Hello\nauthor: Alice\n---\nBody text here.\n",
-        )
-        _write(
-            os.path.join(config_dir, ".dirsql.toml"),
-            """\
-[[table]]
-ddl = "CREATE TABLE posts (title TEXT, author TEXT, body TEXT)"
-glob = "posts/*.md"
-""",
-        )
-        db = DirSQL(config=os.path.join(config_dir, ".dirsql.toml"))
-        await db.ready()
-        results = await db.query("SELECT * FROM posts")
-        assert len(results) == 1
-        assert results[0]["title"] == "Hello"
-        assert results[0]["author"] == "Alice"
-        assert "Body text here." in (results[0]["body"] or "")
-
+    async def it_raises_on_extra_keys_when_strict_true(tmp_dir):
+        """Docs (guide/tables.md "Strict Mode"): strict=True errors on extra keys."""
+        with open(os.path.join(tmp_dir, "a.json"), "w") as f:
+            f.write("{}")
 
-def describe_from_config_strict_mode_gap():
-    @pytest.mark.asyncio
-    async def it_raises_on_extra_keys_when_strict_true(config_dir):
-        """Docs (guide/config.md "Strict Mode"): `strict = true` errors on extra keys."""
-        _write(
-            os.path.join(config_dir, "items", "a.json"),
-            json.dumps({"name": "apple", "color": "red"}),
-        )
-        _write(
-            os.path.join(config_dir, ".dirsql.toml"),
-            """\
-[[table]]
-ddl = "CREATE TABLE items (name TEXT)"
-glob = "items/*.json"
-strict = true
-""",
+        db = DirSQL(
+            tmp_dir,
+            tables=[
+                Table(
+                    ddl="CREATE TABLE items (name TEXT)",
+                    glob="*.json",
+                    extract=lambda path, content: [{"name": "apple", "color": "red"}],
+                    strict=True,
+                ),
+            ],
         )
-        db = DirSQL(config=os.path.join(config_dir, ".dirsql.toml"))
         with pytest.raises(Exception):
             await db.ready()
 
     @pytest.mark.asyncio
-    async def it_allows_exact_match_when_strict_true(config_dir):
-        """Docs (guide/config.md "Strict Mode"): strict mode passes on exact key match."""
-        _write(
-            os.path.join(config_dir, "items", "a.json"),
-            json.dumps({"name": "apple", "color": "red"}),
-        )
-        _write(
-            os.path.join(config_dir, ".dirsql.toml"),
-            """\
-[[table]]
-ddl = "CREATE TABLE items (name TEXT, color TEXT)"
-glob = "items/*.json"
-strict = true
-""",
+    async def it_allows_exact_match_when_strict_true(tmp_dir):
+        """Docs (guide/tables.md "Strict Mode"): strict mode passes on exact key match."""
+        with open(os.path.join(tmp_dir, "a.json"), "w") as f:
+            f.write("{}")
+
+        db = DirSQL(
+            tmp_dir,
+            tables=[
+                Table(
+                    ddl="CREATE TABLE items (name TEXT, color TEXT)",
+                    glob="*.json",
+                    extract=lambda path, content: [{"name": "apple", "color": "red"}],
+                    strict=True,
+                ),
+            ],
         )
-        db = DirSQL(config=os.path.join(config_dir, ".dirsql.toml"))
         await db.ready()
         results = await db.query("SELECT * FROM items")
         assert len(results) == 1