dirsql-plugin-embeddings 0.1.7__tar.gz → 0.1.9__tar.gz

{dirsql_plugin_embeddings-0.1.7 → dirsql_plugin_embeddings-0.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql-plugin-embeddings
-Version: 0.1.7
+Version: 0.1.9
 Summary: First-party dirsql plugin: semantic search via an OpenAI-compatible embeddings endpoint.
 Requires-Python: >=3.10
 Requires-Dist: cachetta>=0.6.15
@@ -10,7 +10,8 @@ Description-Content-Type: text/markdown
 # dirsql-plugin-embeddings
 
 A first-party [`dirsql`](https://github.com/thekevinscott/dirsql) plugin that
-adds **semantic search** over a directory of Markdown files and PDFs. It is the worked
+adds **semantic search** over a directory of documents -- Markdown, plain
+text, reStructuredText and PDFs. It is the worked
 implementation behind the [Search documents by
 meaning](https://thekevinscott.github.io/dirsql/howto/search-by-meaning) how-to,
 swapping that guide's local `model2vec` model for any OpenAI-compatible
@@ -30,23 +31,31 @@ is installed alongside it. The fragment declares:
 
 - the [`sqlite-vec`](https://github.com/asg017/sqlite-vec) extension, for
   `vec_distance_cosine()`;
-- a `documents` table whose `on-file` hook embeds each `**/*.{md,pdf}` file into
-  a TEXT `embedding` column;
+- a `documents` table whose `on-file` hook embeds each
+  `**/*.{md,markdown,mdx,rst,txt,pdf}` file into a TEXT `embedding` column;
 - a `pre-query` hook that embeds the incoming question and emits the
   nearest-neighbor SQL.
 
 Both hooks are console scripts that call the same embedder.
 
-Markdown is read as UTF-8 text; a `.pdf` is read with
+Every matched extension except `.pdf` is read as UTF-8 text; a `.pdf` is read with
 [pypdf](https://pypdf.readthedocs.io), whose per-page extracted text is joined
 and embedded like any other document. The extension check is case-insensitive
 (`.PDF` is a PDF), though the glob above is not — globset matches case-sensitively,
 so an uppercase-suffixed file needs its own `glob` entry to be picked up at all.
 
-A PDF that cannot be parsed aborts the scan rather than being skipped: the hook
-exits non-zero and dirsql reports the path and the pypdf reason. A *scanned*,
-image-only PDF is not a failure — pypdf yields no text, and the file is indexed
-with an empty `text`, exactly like an empty `.md`.
+The glob is an allowlist rather than `**/*` on cost, not correctness. Every
+matched file costs a hook subprocess, and every file the plugin can decode costs
+a billed embedding call — so pointing `**/*` at a tree containing `node_modules`
+or `.git` makes for a slow and expensive scan. The list is what is worth
+embedding; widening it trades money for recall.
+
+A file the plugin cannot read is skipped, not fatal. The hook exits non-zero,
+dirsql names the file on stderr and carries on indexing the rest, and the run
+exits `23` — "completed, some files skipped". From the SDK the same information
+is on `scan_failures()` / `scanFailures()`. A *scanned*, image-only PDF is not a
+failure at all: pypdf yields no text, and the file is indexed with an empty
+`text`, exactly like an empty `.md`.
 
 ## Configuration
 

{dirsql_plugin_embeddings-0.1.7 → dirsql_plugin_embeddings-0.1.9}/README.md

@@ -1,7 +1,8 @@
 # dirsql-plugin-embeddings
 
 A first-party [`dirsql`](https://github.com/thekevinscott/dirsql) plugin that
-adds **semantic search** over a directory of Markdown files and PDFs. It is the worked
+adds **semantic search** over a directory of documents -- Markdown, plain
+text, reStructuredText and PDFs. It is the worked
 implementation behind the [Search documents by
 meaning](https://thekevinscott.github.io/dirsql/howto/search-by-meaning) how-to,
 swapping that guide's local `model2vec` model for any OpenAI-compatible
@@ -21,23 +22,31 @@ is installed alongside it. The fragment declares:
 
 - the [`sqlite-vec`](https://github.com/asg017/sqlite-vec) extension, for
   `vec_distance_cosine()`;
-- a `documents` table whose `on-file` hook embeds each `**/*.{md,pdf}` file into
-  a TEXT `embedding` column;
+- a `documents` table whose `on-file` hook embeds each
+  `**/*.{md,markdown,mdx,rst,txt,pdf}` file into a TEXT `embedding` column;
 - a `pre-query` hook that embeds the incoming question and emits the
   nearest-neighbor SQL.
 
 Both hooks are console scripts that call the same embedder.
 
-Markdown is read as UTF-8 text; a `.pdf` is read with
+Every matched extension except `.pdf` is read as UTF-8 text; a `.pdf` is read with
 [pypdf](https://pypdf.readthedocs.io), whose per-page extracted text is joined
 and embedded like any other document. The extension check is case-insensitive
 (`.PDF` is a PDF), though the glob above is not — globset matches case-sensitively,
 so an uppercase-suffixed file needs its own `glob` entry to be picked up at all.
 
-A PDF that cannot be parsed aborts the scan rather than being skipped: the hook
-exits non-zero and dirsql reports the path and the pypdf reason. A *scanned*,
-image-only PDF is not a failure — pypdf yields no text, and the file is indexed
-with an empty `text`, exactly like an empty `.md`.
+The glob is an allowlist rather than `**/*` on cost, not correctness. Every
+matched file costs a hook subprocess, and every file the plugin can decode costs
+a billed embedding call — so pointing `**/*` at a tree containing `node_modules`
+or `.git` makes for a slow and expensive scan. The list is what is worth
+embedding; widening it trades money for recall.
+
+A file the plugin cannot read is skipped, not fatal. The hook exits non-zero,
+dirsql names the file on stderr and carries on indexing the rest, and the run
+exits `23` — "completed, some files skipped". From the SDK the same information
+is on `scan_failures()` / `scanFailures()`. A *scanned*, image-only PDF is not a
+failure at all: pypdf yields no text, and the file is indexed with an empty
+`text`, exactly like an empty `.md`.
 
 ## Configuration
 

dirsql_plugin_embeddings-0.1.7/e2e-attestations/claude-700-route-pdf.json → dirsql_plugin_embeddings-0.1.9/e2e-attestations/claude-706-widen-glob.json

@@ -1,7 +1,7 @@
 {
   "command": "uv run --with sqlite-vec --with-editable . --with-editable ../../packages/python python -m pytest tests/e2e -x -q",
-  "ran_at": 1785338799,
+  "ran_at": 1785339217,
   "exit_code": 0,
-  "commit": "fcca643ac74c98dd9a6b525681711c720c303250",
-  "branch": "claude/700-route-pdf"
+  "commit": "f96d82085395d4e38bd74992a6ae3c7b029d5f60",
+  "branch": "claude/706-widen-glob"
 }

dirsql_plugin_embeddings-0.1.9/e2e-attestations/claude-725-plugin-glob-comment.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run --with sqlite-vec --with-editable . --with-editable ../../packages/python python -m pytest tests/e2e -q",
+  "ran_at": 1785680099,
+  "exit_code": 0,
+  "commit": "0030917b47975bfa9b4d7412e7f00ca2b760f292",
+  "branch": "claude/725-plugin-glob-comment"
+}

{dirsql_plugin_embeddings-0.1.7 → dirsql_plugin_embeddings-0.1.9}/src/dirsql_plugin_embeddings/dirsql.toml

@@ -15,7 +15,13 @@ entrypoint = "sqlite3_vec_init"
 # The brace is globset alternation, not a dirsql `{name}` placeholder: the core
 # rewrites those to `*`, but only when the braces hold a bare identifier, so a
 # comma-separated list reaches globset intact.
+#
+# An allowlist rather than `**/*` because every matched file costs a hook
+# subprocess and, for anything `read_content` can decode, a billed embedding
+# call. Pointed at a tree holding `node_modules` or `.git`, `**/*` is a slow
+# and expensive scan, not a broken one -- unreadable files are skipped. So the
+# list is what is worth embedding, and widening it trades money for recall.
 [[table]]
 ddl = "CREATE TABLE documents (path TEXT, text TEXT, embedding TEXT)"
-glob = "**/*.{md,pdf}"
+glob = "**/*.{md,markdown,mdx,rst,txt,pdf}"
 on-file = "dirsql-embeddings-on-file {path}"