dirsql-plugin-embeddings 0.1.7__tar.gz → 0.1.8__tar.gz

{dirsql_plugin_embeddings-0.1.7 → dirsql_plugin_embeddings-0.1.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dirsql-plugin-embeddings
-Version: 0.1.7
+Version: 0.1.8
 Summary: First-party dirsql plugin: semantic search via an OpenAI-compatible embeddings endpoint.
 Requires-Python: >=3.10
 Requires-Dist: cachetta>=0.6.15
@@ -10,7 +10,8 @@ Description-Content-Type: text/markdown
 # dirsql-plugin-embeddings
 
 A first-party [`dirsql`](https://github.com/thekevinscott/dirsql) plugin that
-adds **semantic search** over a directory of Markdown files and PDFs. It is the worked
+adds **semantic search** over a directory of documents -- Markdown, plain
+text, reStructuredText and PDFs. It is the worked
 implementation behind the [Search documents by
 meaning](https://thekevinscott.github.io/dirsql/howto/search-by-meaning) how-to,
 swapping that guide's local `model2vec` model for any OpenAI-compatible
@@ -30,19 +31,26 @@ is installed alongside it. The fragment declares:
 
 - the [`sqlite-vec`](https://github.com/asg017/sqlite-vec) extension, for
   `vec_distance_cosine()`;
-- a `documents` table whose `on-file` hook embeds each `**/*.{md,pdf}` file into
-  a TEXT `embedding` column;
+- a `documents` table whose `on-file` hook embeds each
+  `**/*.{md,markdown,mdx,rst,txt,pdf}` file into a TEXT `embedding` column;
 - a `pre-query` hook that embeds the incoming question and emits the
   nearest-neighbor SQL.
 
 Both hooks are console scripts that call the same embedder.
 
-Markdown is read as UTF-8 text; a `.pdf` is read with
+Every matched extension except `.pdf` is read as UTF-8 text; a `.pdf` is read with
 [pypdf](https://pypdf.readthedocs.io), whose per-page extracted text is joined
 and embedded like any other document. The extension check is case-insensitive
 (`.PDF` is a PDF), though the glob above is not — globset matches case-sensitively,
 so an uppercase-suffixed file needs its own `glob` entry to be picked up at all.
 
+The glob is an allowlist rather than `**/*` for a reason worth knowing: a hook
+that exits non-zero aborts the entire scan
+([dirsql#697](https://github.com/thekevinscott/dirsql/issues/697)), and reading a
+PNG as UTF-8 does exactly that. Matching everything would mean one image anywhere
+under the root produces no index at all, so the list stays limited to what the
+plugin can actually read.
+
 A PDF that cannot be parsed aborts the scan rather than being skipped: the hook
 exits non-zero and dirsql reports the path and the pypdf reason. A *scanned*,
 image-only PDF is not a failure — pypdf yields no text, and the file is indexed

{dirsql_plugin_embeddings-0.1.7 → dirsql_plugin_embeddings-0.1.8}/README.md

@@ -1,7 +1,8 @@
 # dirsql-plugin-embeddings
 
 A first-party [`dirsql`](https://github.com/thekevinscott/dirsql) plugin that
-adds **semantic search** over a directory of Markdown files and PDFs. It is the worked
+adds **semantic search** over a directory of documents -- Markdown, plain
+text, reStructuredText and PDFs. It is the worked
 implementation behind the [Search documents by
 meaning](https://thekevinscott.github.io/dirsql/howto/search-by-meaning) how-to,
 swapping that guide's local `model2vec` model for any OpenAI-compatible
@@ -21,19 +22,26 @@ is installed alongside it. The fragment declares:
 
 - the [`sqlite-vec`](https://github.com/asg017/sqlite-vec) extension, for
   `vec_distance_cosine()`;
-- a `documents` table whose `on-file` hook embeds each `**/*.{md,pdf}` file into
-  a TEXT `embedding` column;
+- a `documents` table whose `on-file` hook embeds each
+  `**/*.{md,markdown,mdx,rst,txt,pdf}` file into a TEXT `embedding` column;
 - a `pre-query` hook that embeds the incoming question and emits the
   nearest-neighbor SQL.
 
 Both hooks are console scripts that call the same embedder.
 
-Markdown is read as UTF-8 text; a `.pdf` is read with
+Every matched extension except `.pdf` is read as UTF-8 text; a `.pdf` is read with
 [pypdf](https://pypdf.readthedocs.io), whose per-page extracted text is joined
 and embedded like any other document. The extension check is case-insensitive
 (`.PDF` is a PDF), though the glob above is not — globset matches case-sensitively,
 so an uppercase-suffixed file needs its own `glob` entry to be picked up at all.
 
+The glob is an allowlist rather than `**/*` for a reason worth knowing: a hook
+that exits non-zero aborts the entire scan
+([dirsql#697](https://github.com/thekevinscott/dirsql/issues/697)), and reading a
+PNG as UTF-8 does exactly that. Matching everything would mean one image anywhere
+under the root produces no index at all, so the list stays limited to what the
+plugin can actually read.
+
 A PDF that cannot be parsed aborts the scan rather than being skipped: the hook
 exits non-zero and dirsql reports the path and the pypdf reason. A *scanned*,
 image-only PDF is not a failure — pypdf yields no text, and the file is indexed

dirsql_plugin_embeddings-0.1.7/e2e-attestations/claude-700-route-pdf.json → dirsql_plugin_embeddings-0.1.8/e2e-attestations/claude-706-widen-glob.json

@@ -1,7 +1,7 @@
 {
   "command": "uv run --with sqlite-vec --with-editable . --with-editable ../../packages/python python -m pytest tests/e2e -x -q",
-  "ran_at": 1785338799,
+  "ran_at": 1785339217,
   "exit_code": 0,
-  "commit": "fcca643ac74c98dd9a6b525681711c720c303250",
-  "branch": "claude/700-route-pdf"
+  "commit": "f96d82085395d4e38bd74992a6ae3c7b029d5f60",
+  "branch": "claude/706-widen-glob"
 }

{dirsql_plugin_embeddings-0.1.7 → dirsql_plugin_embeddings-0.1.8}/src/dirsql_plugin_embeddings/dirsql.toml

@@ -15,7 +15,13 @@ entrypoint = "sqlite3_vec_init"
 # The brace is globset alternation, not a dirsql `{name}` placeholder: the core
 # rewrites those to `*`, but only when the braces hold a bare identifier, so a
 # comma-separated list reaches globset intact.
+#
+# An allowlist rather than `**/*`, and not for tidiness: a hook that fails takes
+# the whole scan down (dirsql#697), and reading a PNG as UTF-8 fails. Matching
+# everything would mean one image anywhere under the root yields no index at
+# all. The list is therefore what `read_content` can actually read -- prose, not
+# source -- and widening it is safe only as far as that stays true.
 [[table]]
 ddl = "CREATE TABLE documents (path TEXT, text TEXT, embedding TEXT)"
-glob = "**/*.{md,pdf}"
+glob = "**/*.{md,markdown,mdx,rst,txt,pdf}"
 on-file = "dirsql-embeddings-on-file {path}"