sensemaking 0.9.4 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/README.md +50 -100
  2. package/dist/cjs/cli/check.js +7 -13
  3. package/dist/cjs/cli/check.js.map +1 -1
  4. package/dist/cjs/cli/download.d.cts +3 -0
  5. package/dist/cjs/cli/download.d.ts +3 -0
  6. package/dist/cjs/cli/download.js +214 -0
  7. package/dist/cjs/cli/download.js.map +1 -0
  8. package/dist/cjs/cli/index.d.cts +6 -3
  9. package/dist/cjs/cli/index.d.ts +6 -3
  10. package/dist/cjs/cli/index.js +23 -5
  11. package/dist/cjs/cli/index.js.map +1 -1
  12. package/dist/cjs/cli/init.js +4 -1
  13. package/dist/cjs/cli/init.js.map +1 -1
  14. package/dist/cjs/cli/map.js +2 -2
  15. package/dist/cjs/cli/map.js.map +1 -1
  16. package/dist/cjs/cli/named.js +10 -18
  17. package/dist/cjs/cli/named.js.map +1 -1
  18. package/dist/cjs/cli/path.d.cts +3 -0
  19. package/dist/cjs/cli/path.d.ts +3 -0
  20. package/dist/cjs/cli/path.js +161 -0
  21. package/dist/cjs/cli/path.js.map +1 -0
  22. package/dist/cjs/cli/peek.js +3 -2
  23. package/dist/cjs/cli/peek.js.map +1 -1
  24. package/dist/cjs/cli/related.d.cts +3 -0
  25. package/dist/cjs/cli/related.d.ts +3 -0
  26. package/dist/cjs/cli/related.js +279 -0
  27. package/dist/cjs/cli/related.js.map +1 -0
  28. package/dist/cjs/cli/search.js +3 -7
  29. package/dist/cjs/cli/search.js.map +1 -1
  30. package/dist/cjs/cli/shared.d.cts +3 -1
  31. package/dist/cjs/cli/shared.d.ts +3 -1
  32. package/dist/cjs/cli/shared.js +28 -5
  33. package/dist/cjs/cli/shared.js.map +1 -1
  34. package/dist/cjs/cli/sql.d.cts +3 -0
  35. package/dist/cjs/cli/sql.d.ts +3 -0
  36. package/dist/cjs/cli/{query.js → sql.js} +4 -4
  37. package/dist/cjs/cli/sql.js.map +1 -0
  38. package/dist/cjs/cli/status.js +9 -2
  39. package/dist/cjs/cli/status.js.map +1 -1
  40. package/dist/cjs/cli.js +14 -15
  41. package/dist/cjs/cli.js.map +1 -1
  42. package/dist/cjs/commands.d.cts +12 -4
  43. package/dist/cjs/commands.d.ts +12 -4
  44. package/dist/cjs/commands.js +138 -58
  45. package/dist/cjs/commands.js.map +1 -1
  46. package/dist/cjs/config.d.cts +6 -4
  47. package/dist/cjs/config.d.ts +6 -4
  48. package/dist/cjs/config.js +137 -58
  49. package/dist/cjs/config.js.map +1 -1
  50. package/dist/cjs/db.js +14 -34
  51. package/dist/cjs/db.js.map +1 -1
  52. package/dist/cjs/errors.d.cts +1 -1
  53. package/dist/cjs/errors.d.ts +1 -1
  54. package/dist/cjs/errors.js.map +1 -1
  55. package/dist/cjs/features/embed.d.cts +14 -0
  56. package/dist/cjs/features/embed.d.ts +14 -0
  57. package/dist/cjs/features/embed.js +248 -54
  58. package/dist/cjs/features/embed.js.map +1 -1
  59. package/dist/cjs/features/links.js +7 -16
  60. package/dist/cjs/features/links.js.map +1 -1
  61. package/dist/cjs/features/rank.js +2 -6
  62. package/dist/cjs/features/rank.js.map +1 -1
  63. package/dist/cjs/index.d.cts +1 -1
  64. package/dist/cjs/index.d.ts +1 -1
  65. package/dist/cjs/index.js.map +1 -1
  66. package/dist/cjs/output.js +11 -14
  67. package/dist/cjs/output.js.map +1 -1
  68. package/dist/cjs/progress.js +2 -4
  69. package/dist/cjs/progress.js.map +1 -1
  70. package/dist/cjs/scan.d.cts +1 -0
  71. package/dist/cjs/scan.d.ts +1 -0
  72. package/dist/cjs/scan.js +67 -32
  73. package/dist/cjs/scan.js.map +1 -1
  74. package/dist/cjs/search-error.js +4 -6
  75. package/dist/cjs/search-error.js.map +1 -1
  76. package/dist/cjs/traverse.d.cts +8 -0
  77. package/dist/cjs/traverse.d.ts +8 -0
  78. package/dist/cjs/traverse.js +97 -0
  79. package/dist/cjs/traverse.js.map +1 -0
  80. package/dist/esm/cli/check.js +7 -13
  81. package/dist/esm/cli/check.js.map +1 -1
  82. package/dist/esm/cli/download.d.ts +3 -0
  83. package/dist/esm/cli/download.js +27 -0
  84. package/dist/esm/cli/download.js.map +1 -0
  85. package/dist/esm/cli/index.d.ts +6 -3
  86. package/dist/esm/cli/index.js +13 -7
  87. package/dist/esm/cli/index.js.map +1 -1
  88. package/dist/esm/cli/init.js +4 -1
  89. package/dist/esm/cli/init.js.map +1 -1
  90. package/dist/esm/cli/map.js +3 -2
  91. package/dist/esm/cli/map.js.map +1 -1
  92. package/dist/esm/cli/named.js +10 -16
  93. package/dist/esm/cli/named.js.map +1 -1
  94. package/dist/esm/cli/path.d.ts +3 -0
  95. package/dist/esm/cli/path.js +51 -0
  96. package/dist/esm/cli/path.js.map +1 -0
  97. package/dist/esm/cli/peek.js +4 -2
  98. package/dist/esm/cli/peek.js.map +1 -1
  99. package/dist/esm/cli/related.d.ts +3 -0
  100. package/dist/esm/cli/related.js +26 -0
  101. package/dist/esm/cli/related.js.map +1 -0
  102. package/dist/esm/cli/search.js +2 -5
  103. package/dist/esm/cli/search.js.map +1 -1
  104. package/dist/esm/cli/shared.d.ts +3 -1
  105. package/dist/esm/cli/shared.js +28 -5
  106. package/dist/esm/cli/shared.js.map +1 -1
  107. package/dist/esm/cli/sql.d.ts +3 -0
  108. package/dist/esm/cli/{query.js → sql.js} +4 -4
  109. package/dist/esm/cli/sql.js.map +1 -0
  110. package/dist/esm/cli/status.js +8 -1
  111. package/dist/esm/cli/status.js.map +1 -1
  112. package/dist/esm/cli.js +13 -10
  113. package/dist/esm/cli.js.map +1 -1
  114. package/dist/esm/commands.d.ts +12 -4
  115. package/dist/esm/commands.js +111 -68
  116. package/dist/esm/commands.js.map +1 -1
  117. package/dist/esm/config.d.ts +6 -4
  118. package/dist/esm/config.js +126 -75
  119. package/dist/esm/config.js.map +1 -1
  120. package/dist/esm/db.js +16 -38
  121. package/dist/esm/db.js.map +1 -1
  122. package/dist/esm/errors.d.ts +1 -1
  123. package/dist/esm/errors.js.map +1 -1
  124. package/dist/esm/features/embed.d.ts +14 -0
  125. package/dist/esm/features/embed.js +110 -35
  126. package/dist/esm/features/embed.js.map +1 -1
  127. package/dist/esm/features/links.js +8 -17
  128. package/dist/esm/features/links.js.map +1 -1
  129. package/dist/esm/features/rank.js +2 -6
  130. package/dist/esm/features/rank.js.map +1 -1
  131. package/dist/esm/features/types.js.map +1 -1
  132. package/dist/esm/index.d.ts +1 -1
  133. package/dist/esm/index.js.map +1 -1
  134. package/dist/esm/output.js +13 -16
  135. package/dist/esm/output.js.map +1 -1
  136. package/dist/esm/progress.js +2 -4
  137. package/dist/esm/progress.js.map +1 -1
  138. package/dist/esm/scan.d.ts +1 -0
  139. package/dist/esm/scan.js +41 -34
  140. package/dist/esm/scan.js.map +1 -1
  141. package/dist/esm/search-error.js +4 -6
  142. package/dist/esm/search-error.js.map +1 -1
  143. package/dist/esm/traverse.d.ts +8 -0
  144. package/dist/esm/traverse.js +71 -0
  145. package/dist/esm/traverse.js.map +1 -0
  146. package/package.json +18 -5
  147. package/schema.json +12 -16
  148. package/skills/sense/EXAMPLES.md +13 -22
  149. package/skills/sense/SKILL.md +65 -140
  150. package/skills/sense-setup/EXAMPLES.md +30 -46
  151. package/skills/sense-setup/SKILL.md +22 -83
  152. package/dist/cjs/cli/query.d.cts +0 -3
  153. package/dist/cjs/cli/query.d.ts +0 -3
  154. package/dist/cjs/cli/query.js.map +0 -1
  155. package/dist/esm/cli/query.d.ts +0 -3
  156. package/dist/esm/cli/query.js.map +0 -1
package/README.md CHANGED
@@ -1,34 +1,28 @@
1
1
  # sensemaking
2
2
 
3
- Query and search a tree of markdown notes: SQL over frontmatter and links, ranked search over
4
- the prose — words, links, and meaning fused. A CLI that starts, answers, and exits — no
5
- server, no build step.
3
+ Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step.
6
4
 
7
5
  ## Problem
8
6
 
9
- Working with AI agents produces piles of small notes. Past a few dozen, finding the right ones
10
- means grepping or reading whole folders into context. The structure that makes notes navigable —
11
- frontmatter, wikilinks, headings — is exactly what an agent needs, but nothing exposes it as a
12
- query surface.
7
+ Markdown notes accumulate: research, decisions, meeting notes, agent output. Past a few dozen, finding the ones relevant to what you're doing means grepping or reading whole folders into context. The structure that makes notes navigable (frontmatter, wikilinks, headings) is exactly what a query needs, but nothing exposes it as a query surface.
13
8
 
14
- `sense` indexes all of it into SQLite and reconciles against file timestamps on every query, so
15
- results are never stale and nothing has to be running.
9
+ `sense` indexes all of it into SQLite and reconciles against file timestamps on every query, so results are never stale and nothing has to be running.
16
10
 
17
11
  ## Quick start
18
12
 
19
13
  ```bash
20
14
  npm install -g sensemaking
21
15
  cd your-notes && sense init
16
+ sense download # the embedding model, once per machine; nothing fetches it implicitly
22
17
  ```
23
18
 
24
- Needs Node 22.16 or newer: that is the first release whose built-in SQLite carries FTS5, which
25
- `sense search` indexes prose with.
19
+ Needs Node 22.20 or newer: the first release whose built-in SQLite has both FTS5, which `sense search` indexes prose with, and row-returning `INSERT ... RETURNING`, which `sense path` and `peek` walk the link graph with.
26
20
 
27
21
  ```bash
28
22
  sense map # orient: fields, hub notes, recent changes
29
23
  sense search "revenue OR earnings" --k 10 # locate: words + links + meaning, one ranked list
30
24
  sense peek notes/q3-report.md # structure: outline + links, before reading
31
- sense query "SELECT path FROM frontmatter WHERE has(tags, ?)" urgent
25
+ sense sql "SELECT path FROM frontmatter WHERE has(tags, ?)" urgent
32
26
  ```
33
27
 
34
28
  ## Model
@@ -42,14 +36,9 @@ Every file becomes rows in these tables, plus whatever an enabled feature adds o
42
36
  | `links` | `src`, `target` as written, `dst` resolved (`NULL` = dead link) | graph |
43
37
  | `sections` | heading, `level`, `start_line`, `end_line`, `tokens` estimate | structure |
44
38
 
45
- Results are references path, title, summary, excerpt never file contents. Reading happens
46
- afterward through the filesystem, scoped to the line ranges `peek` returns. This is the
47
- [just-in-time context pattern](https://www.anthropic.com/engineering/effective-context-engineering-for-ai-agents):
48
- the agent holds lightweight identifiers and loads payloads only when needed.
39
+ Results are references (path, title, summary, excerpt), never file contents. Reading happens afterward through the filesystem, scoped to the line ranges `peek` returns. This is the [just-in-time context pattern](https://www.anthropic.com/engineering/effective-context-engineering-for-ai-agents): the agent holds lightweight identifiers and loads payloads only when needed.
49
40
 
50
- Output size is a contract, measured per release ([BENCHMARKING.md](BENCHMARKING.md)): `map` is
51
- fixed-size, a `find` row is tens of tokens, and a `peek` stays flat however large the note is —
52
- so what it saves over reading grows with the file, and a small note is cheaper read whole.
41
+ Output size is a contract, measured per release ([BENCHMARKING.md](BENCHMARKING.md)): `map` is fixed-size, a `search` row is tens of tokens, and a `peek` stays flat however large the note is. What it saves over reading grows with the file; a small note is cheaper to read whole.
53
42
 
54
43
  ```sql
55
44
  -- filter and search compose in one query
@@ -64,88 +53,64 @@ ORDER BY bm25(content, 10.0, 5.0, 1.0) LIMIT 10
64
53
  | command | does |
65
54
  |---|---|
66
55
  | `map` | doc count, frontmatter field coverage, top hubs by link rank, recent changes |
67
- | `search "<text>" [--preset name] [--include glob] [--where "<sql>"] [--k n] [--lexical]` | words + links + vectors, one fused ranked list; `via` labels each row's evidence |
68
- | `peek <path>` | frontmatter + heading outline (`[L143-162, ~380t]`) + links both ways first 20 per list, each with its total; the `links` table has the rest |
69
- | `query "<sql>" [params...]` | ad-hoc SQL over all the tables; `?` binds positional args |
56
+ | `search "<text>" [--preset name] [--include glob] [--exclude glob] [--no-exclude] [--where "<sql>"] [--k n]` | words + links + vectors, one fused ranked list; `via` labels each row's evidence |
57
+ | `peek <path> [--preset name] [--where "<sql>"]` | frontmatter + heading outline (`[L143-162, ~380t]`) + links both ways (first 20 per list, each with its total) |
58
+ | `path <a> <b> [--max-depth n] [--preset name] [--where "<sql>"]` | shortest link chain between two notes, or none within the bound |
59
+ | `related <note> [--k n] [--preset name] [--where "<sql>"]` | notes similar in meaning that `<note>` does not yet link to; reads vectors, so semantic-search cost |
60
+ | `sql "<statement>" [params...]` | ad-hoc SQL over all the tables; `?` binds positional args |
70
61
  | `<name> [params...]` | run a query saved in the config; `--list` names them |
71
62
  | `init` | write a starter `sense.config.json` |
72
63
  | `status` | index location, doc count, per-preset coverage, watcher heartbeat |
73
- | `check` | run every saved query and search, so a broken one fails here instead of mid-task |
64
+ | `check` | run every saved query, so a broken one fails here instead of mid-task |
74
65
  | `rebuild` | delete the cache and re-crawl |
66
+ | `download` | fetch the embedding model named in the config (once per machine; nothing else downloads it) |
75
67
  | `watch` | keep the index warm in the background (optional; see [WATCH.md](WATCH.md)) |
76
68
 
77
- `search` runs one text through every engine its scope has FTS5 word match (BM25-ranked,
78
- bare words AND-join, operators are yours), a personalized-PageRank walk over the link graph,
79
- and vector similarity — fused into one list. `via` labels each row's evidence (`match`,
80
- `link`, `vector`, combinations); `similarity` is the cosine against the best-matching chunk;
81
- `lines` points at the section that earned the row — a direct read range. Rows that only
82
- vectors produced are the "these words aren't in the tree, this is what's near in meaning"
83
- signal. `--preset` picks a named settings bundle from the config, `--lexical` skips vectors
84
- for one command, `--where` filters on frontmatter. `--format json` on any reporting command returns
85
- structured output; `--version` and `--help` do what they say.
69
+ `search` runs one text through every engine its scope has: FTS5 word match (BM25-ranked, bare words AND-join, operators are yours), a personalized-PageRank walk over the link graph, and vector similarity, fused into one list. `via` labels each row's evidence (`match`, `link`, `vector`, combinations); `similarity` is the cosine against the best-matching chunk; `lines` points at the section that earned the row (a direct read range). A `vector`-only row means the search words don't appear in that note; it showed up because the model judged it semantically related. `--preset` picks a named settings bundle from the config, `--where` filters on frontmatter. `--format json` on any reporting command returns structured output; `--version` and `--help` do what they say.
86
70
 
87
71
  ## Config
88
72
 
89
- `sense init` writes `sense.config.json`; discovery walks up from cwd like git
90
- (`--config <path>` overrides). Three keys: **presets** (named, self-contained setting
91
- bundles), **queries** (saved commands), and the version.
73
+ `sense init` writes `sense.config.json`; discovery walks up from cwd like git (`--config <path>` overrides).
92
74
 
93
75
  ```json
94
76
  {
95
77
  "$schema": "https://unpkg.com/sensemaking/schema.json",
96
- "version": 3,
78
+ "version": 4,
97
79
  "presets": {
98
80
  "default": { "include": ["**/*.md"], "k": 10 },
99
- "raw": { "include": ["raw/**/*.md"], "k": 5, "semantic": false }
81
+ "raw": { "include": ["raw/**/*.md"], "k": 5 }
100
82
  },
83
+ "embed": { "model": "minishlab/potion-retrieval-32M", "type": "static" },
101
84
  "queries": {
102
- "dead-links": "SELECT src, target FROM links WHERE dst IS NULL",
103
- "by-tag": "SELECT path, title FROM frontmatter WHERE has(tags, ?) ORDER BY path",
104
- "hot": { "search": "pricing OR billing", "preset": "raw" }
85
+ "dead-links": { "sql": "SELECT src, target FROM links WHERE dst IS NULL" },
86
+ "by-tag": { "sql": "SELECT path, title FROM frontmatter WHERE has(tags, ?) ORDER BY path" },
87
+ "hot": { "search": "pricing OR billing", "preset": "raw" }
105
88
  }
106
89
  }
107
90
  ```
108
91
 
109
- A preset bundles `include`/`exclude` globs (which files), `k` (result count), `semantic`
110
- (vectors, on unless `false`), and `where` (a standing SQL filter). Bare commands use
111
- `default`; `--preset` names another; flags override single fields. **Indexing derives from
112
- the presets**: a file is indexed if any preset includes it, and embedded if any covering
113
- preset has semantic on so a `semantic: false` preset's files cost no vectors, and
114
- `status` shows each preset's coverage. Editing a preset rebuilds the cache and says which
115
- preset caused it.
92
+ | key | holds |
93
+ |---|---|
94
+ | `presets` | named bundles of `include`/`exclude` globs, `k` (result count), `semantic` (vectors for this scope, on unless `false`), `where` (a standing SQL filter). A file is indexed if any preset includes it, embedded if a model is named and some covering preset has `semantic` on; `status` shows each preset's coverage. |
95
+ | `embed` | the model vectors are built with. Naming one gives the tree vectors; omitting the block means none at all, whatever the presets say. `sense download` fetches it. |
96
+ | `queries` | entries runnable as `sense <name>`, each naming the verb it runs: `{ sql }` for SQL (`?` binds positional args) or `{ search }` for a ranked search with its settings baked in, so `sense hot` needs no flags. `sense check` runs them all, so a typo'd column fails at check time instead of mid-task. |
97
+ | `version` | schema version; older configs auto-migrate on load, noted on stderr. |
116
98
 
117
- A query entry is a SQL string (`?` binds positional args) or a saved search with its
118
- settings baked in — `sense hot` needs no flags. `sense check` runs them all, so a typo'd
119
- column fails at check time instead of mid-task. Older config versions auto-migrate on
120
- load, noted on stderr.
99
+ Bare commands use the `default` preset; `--preset` names another; flags override single fields. Editing a preset rebuilds the cache and says which preset caused it.
121
100
 
122
- Vectors use the built-in static model (downloaded to `~/.cache/sensemaking` on first use,
123
- never in the package); an optional top-level `"embed": { "model", "type", "url", "key" }`
124
- block points at any Model2Vec model, local path, or OpenAI-compatible endpoint (Ollama,
125
- LM Studio, hosted). Embedding happens on the first semantic search, with progress.
101
+ Vectors need a model, and nothing downloads it implicitly: `sense download` fetches it once per machine into `$XDG_CACHE_HOME/sensemaking/models` (or `~/.cache/...`), one directory per model, shared by every tree (124 MB, never in the package). `embed.model` is a Hugging Face id, or a path to a directory holding `model.safetensors` and `tokenizer.json`, which `sense download` leaves to you to populate. A preset that asks for vectors without the model is an error naming `sense download`, rather than a quieter result that would make the same search answer differently before and after; a `semantic: false` preset never asks, so it is unaffected. An optional top-level `"embed": { "model", "type", "url", "key" }` block points at any Model2Vec model, local path, or OpenAI-compatible endpoint (Ollama, LM Studio, hosted). Embedding the notes themselves happens on the first semantic search, with progress.
126
102
 
127
- `has(field, value)` is the one custom SQL function: array membership on JSON-array fields,
128
- substring on strings, false on missing keys. Frontmatter parsing is lenient — syntax errors are
129
- per-file warnings and the values are still indexed, so one bad note never costs you the crawl.
103
+ `has(field, value)` is the one custom SQL function: array membership on JSON-array fields, substring on strings, false on missing keys. Frontmatter parsing is lenient: syntax errors are per-file warnings, and the values are still indexed, so one bad note never costs you the crawl.
130
104
 
131
105
  ## Scale
132
106
 
133
- Every query starts with a freshness check against the cache in `.sense/`; only changed files are
134
- re-parsed. What to expect as a tree grows:
107
+ Every query starts with a freshness check against the cache in `.sense/`; only changed files are re-parsed. What to expect as a tree grows:
135
108
 
136
- - **Work is linear in note count** — crawl, reconcile, and the freshness check every invocation
137
- pays. That check is the floor cost of a query and the first thing to watch on a large tree.
138
- - **Output is flat.** `map`, `peek`, and a `find` row cost the same on a small tree as a large
139
- one: context cost is bounded by what you ask for, not by how much there is.
140
- - **Bulk changes are paid by whoever queries next.** `sense watch` moves that re-parse into the
141
- background ([WATCH.md](WATCH.md)) — it changes latency, never answers, since every query
142
- reconciles for itself. `sense rebuild` starts the cache over.
109
+ - **Work is linear in note count.** Crawl, reconcile, and the freshness check are what every invocation pays; that check is the floor cost of a query and the first thing to watch on a large tree.
110
+ - **Output is flat.** `map`, `peek`, and a search row cost the same on a small tree as a large one: context cost is bounded by what you ask for, not by how much there is.
111
+ - **Bulk changes are paid by whoever queries next.** `sense watch` moves that re-parse into the background ([WATCH.md](WATCH.md)): it changes latency, never answers, since every query reconciles for itself. `sense rebuild` starts the cache over.
143
112
 
144
- These are release gates rather than hopes: every release regenerates the numbers on pinned
145
- corpora spanning a 4x range in note count plus a stress tree that packs the worst measured
146
- shapes into one place — a megabyte-scale note, heading-dense outlines, dense link graphs,
147
- hundreds of frontmatter fields. A row that grows faster than linearly, or a token count
148
- that grows at all, blocks the release. Current figures: [BENCHMARKING.md](BENCHMARKING.md).
113
+ These are release gates rather than hopes: every release regenerates the numbers on pinned corpora spanning a 4x range in note count plus a stress tree that packs the worst measured shapes into one place: a megabyte-scale note, heading-dense outlines, dense link graphs, hundreds of frontmatter fields. A row that grows faster than linearly, or a token count that grows at all, blocks the release. Current figures: [BENCHMARKING.md](BENCHMARKING.md).
149
114
 
150
115
  ## For AI agents
151
116
 
@@ -153,40 +118,25 @@ that grows at all, blocks the release. Current figures: [BENCHMARKING.md](BENCHM
153
118
  npx skills add kmalakoff/sensemaking # -g for global, -a claude-code to target
154
119
  ```
155
120
 
156
- Two skills: `sense` for querying a tree what each command is for, FTS5 syntax, reading the
157
- `via`/`score`/`similarity` columns, worked examples — and `sense-setup` for making one, where
158
- features, frontmatter conventions, and note size are decisions with consequences either way.
121
+ Two skills: `sense` for querying a tree (what each command is for, FTS5 syntax, reading the `via`/`score`/`similarity` columns, worked examples) and `sense-setup` for making one, where features, frontmatter conventions, and note size are decisions with consequences either way.
159
122
 
160
123
  ## Prior art
161
124
 
162
- - [Effective context engineering for AI agents](https://www.anthropic.com/engineering/effective-context-engineering-for-ai-agents)
163
- (Anthropic): agents should hold lightweight identifiers file paths, links and load payloads
164
- just in time, because context is a finite resource. The commands implement that pattern as a CLI.
165
- - [llm-wiki](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f) (Karpathy): an
166
- agent-maintained wiki navigated by an `index.md` and links, which he notes needs real search
167
- infrastructure past a few hundred pages. `sense map` derives that index from the notes instead of
168
- maintaining it; `find` is the hybrid local search it calls for.
169
- - Agent memory patterns — llm-wiki's raw/wiki split, Claude Code's dreaming-style nightly
170
- consolidation — are trees of small notes with metadata, links, and layers of differing
171
- authority. sense is the query layer such patterns need (filter by metadata and age, scope
172
- by layer, surface near-duplicates semantically), not an implementation of any one of them.
125
+ - [Effective context engineering for AI agents](https://www.anthropic.com/engineering/effective-context-engineering-for-ai-agents) (Anthropic): agents should hold lightweight identifiers (file paths, links) and load payloads just in time, because context is a finite resource. The commands implement that pattern as a CLI.
126
+ - [llm-wiki](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f) (Karpathy): an agent-maintained wiki navigated by an `index.md` and links, which he notes needs real search infrastructure past a few hundred pages. `sense map` derives that index from the notes instead of maintaining it; `sense search` is the hybrid local search it calls for.
127
+ - Agent memory patterns (llm-wiki's raw/wiki split, Claude Code's dreaming-style nightly consolidation) are trees of small notes with metadata, links, and layers of differing authority. sense is the query layer such patterns need: filter by metadata and age, scope by layer, surface near-duplicates semantically. It isn't an implementation of any one of them.
173
128
 
174
129
  ## Alternatives
175
130
 
176
- - **Obsidian Bases/Dataview** same filters, but only inside the running app; agents can't
177
- query it headless.
178
- - **Index-on-build tools (MarkdownDB)** query a snapshot; `sense` reconciles on every query.
179
- - **Note CLIs (zk)** fixed schema; `sense` filters on arbitrary frontmatter.
180
- - **RAG / vector stores** similarity can't express `WHERE status = 'active'`. Here vectors
181
- are one signal inside `search`: same SQLite file, filters compose, every row labels its
182
- evidence (`via`), and a preset turns vectors off per layer of the tree no second store,
183
- no daemon, no native builds.
184
-
185
- Dependencies: [yaml](https://github.com/eemeli/yaml),
186
- [remove-markdown](https://github.com/zuchka/remove-markdown),
187
- [fast-glob](https://github.com/mrmlnc/fast-glob),
188
- [@huggingface/tokenizers](https://github.com/huggingface/tokenizers.js) (pure JS), and
189
- Node's built-in SQLite. No native builds.
131
+ - **Obsidian Bases/Dataview:** same filters, but only inside the running app; agents can't query it headless.
132
+ - **Index-on-build tools (MarkdownDB):** query a snapshot; `sense` reconciles on every query.
133
+ - **Note CLIs (zk):** fixed schema; `sense` filters on arbitrary frontmatter.
134
+ - **Graph/LSP tools (IWE):** structural queries over a markdown graph via LSP/CLI/MCP, retrieval by structure rather than similarity; no SQL, no vector search.
135
+ - **Markdown vector stores (markdown-vdb):** hybrid BM25 + vector search over markdown files, no frontmatter filtering; `sense` treats vectors as one signal alongside SQL, not the whole store.
136
+ - **RAG / vector stores:** similarity can't express `WHERE status = 'active'`. Here vectors are one signal inside `search`: same SQLite file, filters compose, every row labels its evidence (`via`), and a preset turns vectors off per layer of the tree. No second store, no daemon, no native builds.
137
+ - **Document-OS apps (Anytype, Logseq, SilverBullet, Capacities):** full applications with their own UI and storage. `sense` is headless: your files stay files, there's no app to run.
138
+
139
+ Dependencies: [yaml](https://github.com/eemeli/yaml), [remove-markdown](https://github.com/zuchka/remove-markdown), [@huggingface/tokenizers](https://github.com/huggingface/tokenizers.js) (pure JS), and Node's built-in SQLite. No native builds.
190
140
 
191
141
  ## License
192
142
 
@@ -219,16 +219,9 @@ function _ts_generator(thisArg, body) {
219
219
  };
220
220
  }
221
221
  }
222
- // A saved query that returns zero rows looks identical to a true empty result, so a broken
223
- // one can sit unnoticed -- one field report had `WHERE flag = 'true'` against a numeric
224
- // column reading as "nothing tagged yet" for hours. Preparing each query catches syntax and
225
- // unknown-column errors without executing it; queries that take no parameters are also run
226
- // for a row count. Parameterised queries are validated but not counted: inventing arguments
227
- // would report a count for a query nobody ran. Saved searches run lexically with k=1 -- that
228
- // validates the `where` fragment, the FTS5 terms, and the `preset` name (resolveSearch throws
229
- // on an unknown one, the same breakage class prepare() catches for SQL strings) -- but never
230
- // semantically (a semantic pass costs model time and, on api trees, network). Whether a
231
- // result being empty is good or bad is the reader's judgment -- there is no assertion path.
222
+ // A broken query and an empty one look identical, so prepare each to catch syntax and column
223
+ // errors. Parameterised ones are not run (inventing arguments would report a bogus count);
224
+ // searches probe at k=1 without vectors. Empty results are the reader's judgment, not a fail.
232
225
  var check = function check(ctx) {
233
226
  return _async_to_generator(function() {
234
227
  var _cfg_queries, values, format, cfg, _open, db, warnings, rows, failed, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _step_value, name, entry, _sql_match, count, err, sql, params, statement, count1, err1;
@@ -278,7 +271,8 @@ var check = function check(ctx) {
278
271
  where: entry.where,
279
272
  preset: entry.preset,
280
273
  include: entry.include,
281
- semantic: false
274
+ exclude: entry.exclude,
275
+ probe: true
282
276
  })
283
277
  ];
284
278
  case 4:
@@ -287,7 +281,7 @@ var check = function check(ctx) {
287
281
  query: name,
288
282
  params: '—',
289
283
  rows: '—',
290
- status: count === 0 ? 'ok, but matches 0 notes (lexical probe)' : 'ok (probe run with k=1)'
284
+ status: count === 0 ? 'ok, but matches 0 notes (words and links only)' : 'ok (probe run with k=1)'
291
285
  });
292
286
  return [
293
287
  3,
@@ -312,7 +306,7 @@ var check = function check(ctx) {
312
306
  8
313
307
  ];
314
308
  case 7:
315
- sql = typeof entry === 'string' ? entry : entry.sql;
309
+ sql = entry.sql;
316
310
  params = ((_sql_match = sql.match(/\?/g)) !== null && _sql_match !== void 0 ? _sql_match : []).length;
317
311
  try {
318
312
  statement = db.prepare(sql);
@@ -1 +1 @@
1
- {"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/check.ts"],"sourcesContent":["import { search } from '../commands.ts';\nimport { open } from '../db.ts';\nimport type { Row } from '../output.ts';\nimport { printRows } from '../output.ts';\nimport { USAGE } from './index.ts';\nimport { CONFIG, FORMAT, formatOf, parse, printWarnings } from './shared.ts';\nimport type { Command } from './types.ts';\n\n// A saved query that returns zero rows looks identical to a true empty result, so a broken\n// one can sit unnoticed -- one field report had `WHERE flag = 'true'` against a numeric\n// column reading as \"nothing tagged yet\" for hours. Preparing each query catches syntax and\n// unknown-column errors without executing it; queries that take no parameters are also run\n// for a row count. Parameterised queries are validated but not counted: inventing arguments\n// would report a count for a query nobody ran. Saved searches run lexically with k=1 -- that\n// validates the `where` fragment, the FTS5 terms, and the `preset` name (resolveSearch throws\n// on an unknown one, the same breakage class prepare() catches for SQL strings) -- but never\n// semantically (a semantic pass costs model time and, on api trees, network). Whether a\n// result being empty is good or bad is the reader's judgment -- there is no assertion path.\nconst check: Command = async (ctx) => {\n const { values } = parse(ctx.argv, `usage: ${ctx.name} ${USAGE.check}`, { ...FORMAT, ...CONFIG });\n const format = formatOf(values);\n const cfg = ctx.resolveConfig(values.config as string | undefined);\n const { db, warnings } = open(cfg);\n printWarnings(warnings);\n\n const rows: Row[] = [];\n let failed = 0;\n for (const [name, entry] of Object.entries(cfg.queries ?? {})) {\n if (typeof entry === 'object' && entry !== null && 'search' in entry) {\n try {\n const count = (await search(db, cfg, entry.search, { k: 1, where: entry.where, preset: entry.preset, include: entry.include, semantic: false })).length;\n rows.push({ query: name, params: '—', rows: '—', status: count === 0 ? 'ok, but matches 0 notes (lexical probe)' : 'ok (probe run with k=1)' });\n } catch (err) {\n failed++;\n rows.push({ query: name, params: '—', rows: '—', status: `FAILED: ${(err as Error).message}` });\n }\n continue;\n }\n const sql = typeof entry === 'string' ? entry : entry.sql;\n const params = (sql.match(/\\?/g) ?? []).length;\n try {\n const statement = db.prepare(sql);\n if (params > 0) {\n rows.push({ query: name, params, rows: '—', status: 'ok (not run: needs parameters)' });\n continue;\n }\n const count = (statement.all() as unknown[]).length;\n rows.push({ query: name, params, rows: count, status: count === 0 ? 'ok, but returns 0 rows' : 'ok' });\n } catch (err) {\n failed++;\n rows.push({ query: name, params, rows: '—', status: `FAILED: ${(err as Error).message}` });\n }\n }\n\n db.close();\n if (rows.length === 0) {\n console.log('no saved queries in config');\n return;\n }\n printRows(rows, format);\n if (failed > 0) process.exit(1);\n};\nexport default check;\n"],"names":["check","ctx","cfg","values","format","open","db","warnings","rows","failed","name","entry","sql","count","err","params","statement","parse","argv","USAGE","FORMAT","CONFIG","formatOf","resolveConfig","config","printWarnings","Object","entries","queries","search","k","where","preset","include","semantic","length","push","query","status","message","match","prepare","all","close","console","log","printRows","process","exit"],"mappings":";;;;+BA8DA;;;eAAA;;;0BA9DuB;oBACF;wBAEK;uBACJ;wBACyC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG/D,2FAA2F;AAC3F,wFAAwF;AACxF,4FAA4F;AAC5F,2FAA2F;AAC3F,4FAA4F;AAC5F,6FAA6F;AAC7F,8FAA8F;AAC9F,6FAA6F;AAC7F,wFAAwF;AACxF,4FAA4F;AAC5F,IAAMA,QAAiB,eAAOC;;YASeC,cARnCC,QACFC,QACAF,KACmBG,OAAjBC,IAAIC,UAGNC,MACFC,QACC,2BAAA,mBAAA,gBAAA,WAAA,oBAAOC,MAAMC,OAYAC,YATNC,OAECC,KAMLF,KACAG,QAEEC,WAKAH;;;;oBA3BFV,SAAWc,IAAAA,eAAK,EAAChB,IAAIiB,IAAI,EAAE,AAAC,UAAqBC,OAAZlB,IAAIS,IAAI,EAAC,KAAe,OAAZS,cAAK,CAACnB,KAAK,GAAI,mBAAKoB,gBAAM,EAAKC,gBAAM,GAAtFlB;oBACFC,SAASkB,IAAAA,kBAAQ,EAACnB;oBAClBD,MAAMD,IAAIsB,aAAa,CAACpB,OAAOqB,MAAM;oBAClBnB,QAAAA,IAAAA,UAAI,EAACH,MAAtBI,KAAiBD,MAAjBC,IAAIC,WAAaF,MAAbE;oBACZkB,IAAAA,uBAAa,EAAClB;oBAERC;oBACFC,SAAS;oBACR,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAuBiB,OAAOC,OAAO,EAACzB,eAAAA,IAAI0B,OAAO,cAAX1B,0BAAAA,eAAe,CAAC;;;2BAAtD,6BAAA,QAAA;;;;mDAAA,iBAAOQ,uBAAMC;yBACZ,CAAA,CAAA,OAAOA,sCAAP,SAAOA,MAAI,MAAM,YAAYA,UAAU,QAAQ,YAAYA,KAAI,GAA/D;;;;;;;;;;;;oBAEe;;wBAAMkB,IAAAA,kBAAM,EAACvB,IAAIJ,KAAKS,MAAMkB,MAAM,EAAE;4BAAEC,GAAG;4BAAGC,OAAOpB,MAAMoB,KAAK;4BAAEC,QAAQrB,MAAMqB,MAAM;4BAAEC,SAAStB,MAAMsB,OAAO;4BAAEC,UAAU;wBAAM;;;oBAAvIrB,QAAQ,AAAC,cAAkIsB,MAAM;oBACvJ3B,KAAK4B,IAAI,CAAC;wBAAEC,OAAO3B;wBAAMK,QAAQ;wBAAKP,MAAM;wBAAK8B,QAAQzB,UAAU,IAAI,4CAA4C;oBAA0B;;;;;;oBACtIC;oBACPL;oBACAD,KAAK4B,IAAI,CAAC;wBAAEC,OAAO3B;wBAAMK,QAAQ;wBAAKP,MAAM;wBAAK8B,QAAQ,AAAC,WAAiC,OAAvB,AAACxB,IAAcyB,OAAO;oBAAG;;;;;;oBAE/F;;;;;oBAEI3B,MAAM,OAAOD,UAAU,WAAWA,QAAQA,MAAMC,GAAG;oBACnDG,SAAS,EAACH,aAAAA,IAAI4B,KAAK,CAAC,oBAAV5B,wBAAAA,iBAAwBuB,MAAM;oBAC9C,IAAI;wBACInB,YAAYV,GAAGmC,OAAO,CAAC7B;wBAC7B,IAAIG,SAAS,GAAG;4BACdP,KAAK4B,IAAI,CAAC;gCAAEC,OAAO3B;gCAAMK,QAAAA;gCAAQP,MAAM;gCAAK8B,QAAQ;4BAAiC;4BACrF;;;;wBACF;wBACMzB,SAAQ,AAACG,UAAU0B,GAAG,GAAiBP,MAAM;wBACnD3B,KAAK4B,IAAI,CAAC;4BAAEC,OAAO3B;4BAAMK,QAAAA;4BAAQP,MAAMK;4BAAOyB,QAAQzB,WAAU,IAAI,2BAA2B;wBAAK;oBACtG,EAAE,OAAOC,KAAK;wBACZL;wBACAD,KAAK4B,IAAI,CAAC;4BAAEC,OAAO3B;4BAAMK,QAAAA;4BAAQP,MAAM;4BAAK8B,QAAQ,AAAC,WAAiC,OAAvB,AAACxB,IAAcyB,OAAO;wBAAG;oBAC1F;;;oBAxBG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBA2BLjC,GAAGqC,KAAK;oBACR,IAAInC,KAAK2B,MAAM,KAAK,GAAG;wBACrBS,QAAQC,GAAG,CAAC;wBACZ;;;oBACF;oBACAC,IAAAA,mBAAS,EAACtC,MAAMJ;oBAChB,IAAIK,SAAS,GAAGsC,QAAQC,IAAI,CAAC;;;;;;IAC/B;;IACA,WAAehD"}
1
+ {"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/check.ts"],"sourcesContent":["import { search } from '../commands.ts';\nimport { open } from '../db.ts';\nimport type { Row } from '../output.ts';\nimport { printRows } from '../output.ts';\nimport { USAGE } from './index.ts';\nimport { CONFIG, FORMAT, formatOf, parse, printWarnings } from './shared.ts';\nimport type { Command } from './types.ts';\n\n// A broken query and an empty one look identical, so prepare each to catch syntax and column\n// errors. Parameterised ones are not run (inventing arguments would report a bogus count);\n// searches probe at k=1 without vectors. Empty results are the reader's judgment, not a fail.\nconst check: Command = async (ctx) => {\n const { values } = parse(ctx.argv, `usage: ${ctx.name} ${USAGE.check}`, { ...FORMAT, ...CONFIG });\n const format = formatOf(values);\n const cfg = ctx.resolveConfig(values.config as string | undefined);\n const { db, warnings } = open(cfg);\n printWarnings(warnings);\n\n const rows: Row[] = [];\n let failed = 0;\n for (const [name, entry] of Object.entries(cfg.queries ?? {})) {\n if (typeof entry === 'object' && entry !== null && 'search' in entry) {\n try {\n const count = (await search(db, cfg, entry.search, { k: 1, where: entry.where, preset: entry.preset, include: entry.include, exclude: entry.exclude, probe: true })).length;\n rows.push({ query: name, params: '—', rows: '—', status: count === 0 ? 'ok, but matches 0 notes (words and links only)' : 'ok (probe run with k=1)' });\n } catch (err) {\n failed++;\n rows.push({ query: name, params: '—', rows: '—', status: `FAILED: ${(err as Error).message}` });\n }\n continue;\n }\n const sql = entry.sql;\n const params = (sql.match(/\\?/g) ?? []).length;\n try {\n const statement = db.prepare(sql);\n if (params > 0) {\n rows.push({ query: name, params, rows: '—', status: 'ok (not run: needs parameters)' });\n continue;\n }\n const count = (statement.all() as unknown[]).length;\n rows.push({ query: name, params, rows: count, status: count === 0 ? 'ok, but returns 0 rows' : 'ok' });\n } catch (err) {\n failed++;\n rows.push({ query: name, params, rows: '—', status: `FAILED: ${(err as Error).message}` });\n }\n }\n\n db.close();\n if (rows.length === 0) {\n console.log('no saved queries in config');\n return;\n }\n printRows(rows, format);\n if (failed > 0) process.exit(1);\n};\nexport default check;\n"],"names":["check","ctx","cfg","values","format","open","db","warnings","rows","failed","name","entry","sql","count","err","params","statement","parse","argv","USAGE","FORMAT","CONFIG","formatOf","resolveConfig","config","printWarnings","Object","entries","queries","search","k","where","preset","include","exclude","probe","length","push","query","status","message","match","prepare","all","close","console","log","printRows","process","exit"],"mappings":";;;;+BAuDA;;;eAAA;;;0BAvDuB;oBACF;wBAEK;uBACJ;wBACyC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG/D,6FAA6F;AAC7F,2FAA2F;AAC3F,8FAA8F;AAC9F,IAAMA,QAAiB,eAAOC;;YASeC,cARnCC,QACFC,QACAF,KACmBG,OAAjBC,IAAIC,UAGNC,MACFC,QACC,2BAAA,mBAAA,gBAAA,WAAA,oBAAOC,MAAMC,OAYAC,YATNC,OAECC,KAMLF,KACAG,QAEEC,WAKAH;;;;oBA3BFV,SAAWc,IAAAA,eAAK,EAAChB,IAAIiB,IAAI,EAAE,AAAC,UAAqBC,OAAZlB,IAAIS,IAAI,EAAC,KAAe,OAAZS,cAAK,CAACnB,KAAK,GAAI,mBAAKoB,gBAAM,EAAKC,gBAAM,GAAtFlB;oBACFC,SAASkB,IAAAA,kBAAQ,EAACnB;oBAClBD,MAAMD,IAAIsB,aAAa,CAACpB,OAAOqB,MAAM;oBAClBnB,QAAAA,IAAAA,UAAI,EAACH,MAAtBI,KAAiBD,MAAjBC,IAAIC,WAAaF,MAAbE;oBACZkB,IAAAA,uBAAa,EAAClB;oBAERC;oBACFC,SAAS;oBACR,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAuBiB,OAAOC,OAAO,EAACzB,eAAAA,IAAI0B,OAAO,cAAX1B,0BAAAA,eAAe,CAAC;;;2BAAtD,6BAAA,QAAA;;;;mDAAA,iBAAOQ,uBAAMC;yBACZ,CAAA,CAAA,OAAOA,sCAAP,SAAOA,MAAI,MAAM,YAAYA,UAAU,QAAQ,YAAYA,KAAI,GAA/D;;;;;;;;;;;;oBAEe;;wBAAMkB,IAAAA,kBAAM,EAACvB,IAAIJ,KAAKS,MAAMkB,MAAM,EAAE;4BAAEC,GAAG;4BAAGC,OAAOpB,MAAMoB,KAAK;4BAAEC,QAAQrB,MAAMqB,MAAM;4BAAEC,SAAStB,MAAMsB,OAAO;4BAAEC,SAASvB,MAAMuB,OAAO;4BAAEC,OAAO;wBAAK;;;oBAA3JtB,QAAQ,AAAC,cAAsJuB,MAAM;oBAC3K5B,KAAK6B,IAAI,CAAC;wBAAEC,OAAO5B;wBAAMK,QAAQ;wBAAKP,MAAM;wBAAK+B,QAAQ1B,UAAU,IAAI,mDAAmD;oBAA0B;;;;;;oBAC7IC;oBACPL;oBACAD,KAAK6B,IAAI,CAAC;wBAAEC,OAAO5B;wBAAMK,QAAQ;wBAAKP,MAAM;wBAAK+B,QAAQ,AAAC,WAAiC,OAAvB,AAACzB,IAAc0B,OAAO;oBAAG;;;;;;oBAE/F;;;;;oBAEI5B,MAAMD,MAAMC,GAAG;oBACfG,SAAS,EAACH,aAAAA,IAAI6B,KAAK,CAAC,oBAAV7B,wBAAAA,iBAAwBwB,MAAM;oBAC9C,IAAI;wBACIpB,YAAYV,GAAGoC,OAAO,CAAC9B;wBAC7B,IAAIG,SAAS,GAAG;4BACdP,KAAK6B,IAAI,CAAC;gCAAEC,OAAO5B;gCAAMK,QAAAA;gCAAQP,MAAM;gCAAK+B,QAAQ;4BAAiC;4BACrF;;;;wBACF;wBACM1B,SAAQ,AAACG,UAAU2B,GAAG,GAAiBP,MAAM;wBACnD5B,KAAK6B,IAAI,CAAC;4BAAEC,OAAO5B;4BAAMK,QAAAA;4BAAQP,MAAMK;4BAAO0B,QAAQ1B,WAAU,IAAI,2BAA2B;wBAAK;oBACtG,EAAE,OAAOC,KAAK;wBACZL;wBACAD,KAAK6B,IAAI,CAAC;4BAAEC,OAAO5B;4BAAMK,QAAAA;4BAAQP,MAAM;4BAAK+B,QAAQ,AAAC,WAAiC,OAAvB,AAACzB,IAAc0B,OAAO;wBAAG;oBAC1F;;;oBAxBG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBA2BLlC,GAAGsC,KAAK;oBACR,IAAIpC,KAAK4B,MAAM,KAAK,GAAG;wBACrBS,QAAQC,GAAG,CAAC;wBACZ;;;oBACF;oBACAC,IAAAA,mBAAS,EAACvC,MAAMJ;oBAChB,IAAIK,SAAS,GAAGuC,QAAQC,IAAI,CAAC;;;;;;IAC/B;;IACA,WAAejD"}
@@ -0,0 +1,3 @@
1
+ import type { Command } from './types.js';
2
+ declare const download: Command;
3
+ export default download;
@@ -0,0 +1,3 @@
1
+ import type { Command } from './types.js';
2
+ declare const download: Command;
3
+ export default download;
@@ -0,0 +1,214 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", {
3
+ value: true
4
+ });
5
+ Object.defineProperty(exports, "default", {
6
+ enumerable: true,
7
+ get: function() {
8
+ return _default;
9
+ }
10
+ });
11
+ var _configts = require("../config.js");
12
+ var _errorsts = require("../errors.js");
13
+ var _embedts = require("../features/embed.js");
14
+ var _indexts = require("./index.js");
15
+ var _sharedts = require("./shared.js");
16
+ function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
17
+ try {
18
+ var info = gen[key](arg);
19
+ var value = info.value;
20
+ } catch (error) {
21
+ reject(error);
22
+ return;
23
+ }
24
+ if (info.done) {
25
+ resolve(value);
26
+ } else {
27
+ Promise.resolve(value).then(_next, _throw);
28
+ }
29
+ }
30
+ function _async_to_generator(fn) {
31
+ return function() {
32
+ var self = this, args = arguments;
33
+ return new Promise(function(resolve, reject) {
34
+ var gen = fn.apply(self, args);
35
+ function _next(value) {
36
+ asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
37
+ }
38
+ function _throw(err) {
39
+ asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
40
+ }
41
+ _next(undefined);
42
+ });
43
+ };
44
+ }
45
+ function _define_property(obj, key, value) {
46
+ if (key in obj) {
47
+ Object.defineProperty(obj, key, {
48
+ value: value,
49
+ enumerable: true,
50
+ configurable: true,
51
+ writable: true
52
+ });
53
+ } else {
54
+ obj[key] = value;
55
+ }
56
+ return obj;
57
+ }
58
+ function _object_spread(target) {
59
+ for(var i = 1; i < arguments.length; i++){
60
+ var source = arguments[i] != null ? arguments[i] : {};
61
+ var ownKeys = Object.keys(source);
62
+ if (typeof Object.getOwnPropertySymbols === "function") {
63
+ ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
64
+ return Object.getOwnPropertyDescriptor(source, sym).enumerable;
65
+ }));
66
+ }
67
+ ownKeys.forEach(function(key) {
68
+ _define_property(target, key, source[key]);
69
+ });
70
+ }
71
+ return target;
72
+ }
73
+ function _ts_generator(thisArg, body) {
74
+ var f, y, t, _ = {
75
+ label: 0,
76
+ sent: function() {
77
+ if (t[0] & 1) throw t[1];
78
+ return t[1];
79
+ },
80
+ trys: [],
81
+ ops: []
82
+ }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
83
+ return d(g, "next", {
84
+ value: verb(0)
85
+ }), d(g, "throw", {
86
+ value: verb(1)
87
+ }), d(g, "return", {
88
+ value: verb(2)
89
+ }), typeof Symbol === "function" && d(g, Symbol.iterator, {
90
+ value: function() {
91
+ return this;
92
+ }
93
+ }), g;
94
+ function verb(n) {
95
+ return function(v) {
96
+ return step([
97
+ n,
98
+ v
99
+ ]);
100
+ };
101
+ }
102
+ function step(op) {
103
+ if (f) throw new TypeError("Generator is already executing.");
104
+ while(g && (g = 0, op[0] && (_ = 0)), _)try {
105
+ if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
106
+ if (y = 0, t) op = [
107
+ op[0] & 2,
108
+ t.value
109
+ ];
110
+ switch(op[0]){
111
+ case 0:
112
+ case 1:
113
+ t = op;
114
+ break;
115
+ case 4:
116
+ _.label++;
117
+ return {
118
+ value: op[1],
119
+ done: false
120
+ };
121
+ case 5:
122
+ _.label++;
123
+ y = op[1];
124
+ op = [
125
+ 0
126
+ ];
127
+ continue;
128
+ case 7:
129
+ op = _.ops.pop();
130
+ _.trys.pop();
131
+ continue;
132
+ default:
133
+ if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
134
+ _ = 0;
135
+ continue;
136
+ }
137
+ if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
138
+ _.label = op[1];
139
+ break;
140
+ }
141
+ if (op[0] === 6 && _.label < t[1]) {
142
+ _.label = t[1];
143
+ t = op;
144
+ break;
145
+ }
146
+ if (t && _.label < t[2]) {
147
+ _.label = t[2];
148
+ _.ops.push(op);
149
+ break;
150
+ }
151
+ if (t[2]) _.ops.pop();
152
+ _.trys.pop();
153
+ continue;
154
+ }
155
+ op = body.call(thisArg, _);
156
+ } catch (e) {
157
+ op = [
158
+ 6,
159
+ e
160
+ ];
161
+ y = 0;
162
+ } finally{
163
+ f = t = 0;
164
+ }
165
+ if (op[0] & 5) throw op[1];
166
+ return {
167
+ value: op[0] ? op[1] : void 0,
168
+ done: true
169
+ };
170
+ }
171
+ }
172
+ // The one command that fetches weights. Nothing else touches the network for them, so a
173
+ // query that reads like a query never spends minutes pulling 124 MB. Asks nothing, so there
174
+ // is no prompt for CI to bypass; running it twice is a no-op.
175
+ var download = function download(ctx) {
176
+ return _async_to_generator(function() {
177
+ var values, cfg, e, dir;
178
+ return _ts_generator(this, function(_state) {
179
+ switch(_state.label){
180
+ case 0:
181
+ values = (0, _sharedts.parse)(ctx.argv, "usage: ".concat(ctx.name, " ").concat(_indexts.USAGE.download), _object_spread({}, _sharedts.CONFIG)).values;
182
+ cfg = ctx.resolveConfig(values.config);
183
+ e = (0, _configts.embedConfig)(cfg);
184
+ if (!e) throw new _errorsts.SenseError('EMBED_DISABLED', 'no preset asks for vectors, so there is no model to download');
185
+ if (e.type === 'api') {
186
+ console.log('embed type is "api" ('.concat(e.url, "), so there is nothing to download; ").concat(ctx.name, " search reaches it over the network"));
187
+ return [
188
+ 2
189
+ ];
190
+ }
191
+ if ((0, _embedts.modelPresent)(cfg)) {
192
+ console.log("".concat(e.model, " is already available in ").concat((0, _embedts.modelDir)(e.model)));
193
+ return [
194
+ 2
195
+ ];
196
+ }
197
+ return [
198
+ 4,
199
+ (0, _embedts.downloadModel)(e.model, function(file, into) {
200
+ return console.error("fetching ".concat(e.model, "/").concat(file, " into ").concat(into));
201
+ })
202
+ ];
203
+ case 1:
204
+ dir = _state.sent();
205
+ console.log("downloaded ".concat(e.model, " into ").concat(dir));
206
+ return [
207
+ 2
208
+ ];
209
+ }
210
+ });
211
+ })();
212
+ };
213
+ var _default = download;
214
+ /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/download.ts"],"sourcesContent":["import { embedConfig } from '../config.ts';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, modelDir, modelPresent } from '../features/embed.ts';\nimport { USAGE } from './index.ts';\nimport { CONFIG, parse } from './shared.ts';\nimport type { Command } from './types.ts';\n\n// The one command that fetches weights. Nothing else touches the network for them, so a\n// query that reads like a query never spends minutes pulling 124 MB. Asks nothing, so there\n// is no prompt for CI to bypass; running it twice is a no-op.\nconst download: Command = async (ctx) => {\n const { values } = parse(ctx.argv, `usage: ${ctx.name} ${USAGE.download}`, { ...CONFIG });\n const cfg = ctx.resolveConfig(values.config as string | undefined);\n const e = embedConfig(cfg);\n if (!e) throw new SenseError('EMBED_DISABLED', 'no preset asks for vectors, so there is no model to download');\n\n if (e.type === 'api') {\n console.log(`embed type is \"api\" (${e.url}), so there is nothing to download; ${ctx.name} search reaches it over the network`);\n return;\n }\n if (modelPresent(cfg)) {\n console.log(`${e.model} is already available in ${modelDir(e.model)}`);\n return;\n }\n const dir = await downloadModel(e.model, (file, into) => console.error(`fetching ${e.model}/${file} into ${into}`));\n console.log(`downloaded ${e.model} into ${dir}`);\n};\nexport default download;\n"],"names":["download","ctx","values","cfg","e","dir","parse","argv","USAGE","name","CONFIG","resolveConfig","config","embedConfig","SenseError","type","console","log","url","modelPresent","modelDir","model","downloadModel","file","into","error"],"mappings":";;;;+BA2BA;;;eAAA;;;wBA3B4B;wBACD;uBAC2B;uBAChC;wBACQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG9B,wFAAwF;AACxF,4FAA4F;AAC5F,8DAA8D;AAC9D,IAAMA,WAAoB,kBAAOC;;YACvBC,QACFC,KACAC,GAWAC;;;;oBAbEH,SAAWI,IAAAA,eAAK,EAACL,IAAIM,IAAI,EAAE,AAAC,UAAqBC,OAAZP,IAAIQ,IAAI,EAAC,KAAkB,OAAfD,cAAK,CAACR,QAAQ,GAAI,mBAAKU,gBAAM,GAA9ER;oBACFC,MAAMF,IAAIU,aAAa,CAACT,OAAOU,MAAM;oBACrCR,IAAIS,IAAAA,qBAAW,EAACV;oBACtB,IAAI,CAACC,GAAG,MAAM,IAAIU,oBAAU,CAAC,kBAAkB;oBAE/C,IAAIV,EAAEW,IAAI,KAAK,OAAO;wBACpBC,QAAQC,GAAG,CAAC,AAAC,wBAAmEhB,OAA5CG,EAAEc,GAAG,EAAC,wCAA+C,OAATjB,IAAIQ,IAAI,EAAC;wBACzF;;;oBACF;oBACA,IAAIU,IAAAA,qBAAY,EAAChB,MAAM;wBACrBa,QAAQC,GAAG,CAAC,AAAC,GAAqCG,OAAnChB,EAAEiB,KAAK,EAAC,6BAA6C,OAAlBD,IAAAA,iBAAQ,EAAChB,EAAEiB,KAAK;wBAClE;;;oBACF;oBACY;;wBAAMC,IAAAA,sBAAa,EAAClB,EAAEiB,KAAK,EAAE,SAACE,MAAMC;mCAASR,QAAQS,KAAK,CAAC,AAAC,YAAsBF,OAAXnB,EAAEiB,KAAK,EAAC,KAAgBG,OAAbD,MAAK,UAAa,OAALC;;;;oBAArGnB,MAAM;oBACZW,QAAQC,GAAG,CAAC,AAAC,cAA6BZ,OAAhBD,EAAEiB,KAAK,EAAC,UAAY,OAAJhB;;;;;;IAC5C;;IACA,WAAeL"}
@@ -5,10 +5,13 @@ export declare const USAGE: {
5
5
  readonly status: "status [--format table|json] [--config path]";
6
6
  readonly check: "check [--format table|json] [--config path]";
7
7
  readonly rebuild: "rebuild [--config path]";
8
- readonly query: "query \"<sql>\" [params...] [--format table|json] [--config path]";
9
- readonly search: "search \"<terms>\" [--preset name] [--include glob ...] [--where \"<sql>\"] [--k n] [--lexical] [--format table|json] [--config path]";
8
+ readonly download: "download [--config path]";
9
+ readonly sql: "sql \"<statement>\" [params...] [--format table|json] [--config path]";
10
+ readonly search: "search \"<terms>\" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json] [--config path]";
10
11
  readonly map: "map [--format table|json] [--config path]";
11
- readonly peek: "peek <path> [--format table|json] [--config path]";
12
+ readonly peek: "peek <path> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--format table|json] [--config path]";
13
+ readonly path: "path <a> <b> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--max-depth n] [--format table|json] [--config path]";
14
+ readonly related: "related <note> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json] [--config path]";
12
15
  };
13
16
  export declare const COMMANDS: Record<string, () => Promise<{
14
17
  default: Command;
@@ -5,10 +5,13 @@ export declare const USAGE: {
5
5
  readonly status: "status [--format table|json] [--config path]";
6
6
  readonly check: "check [--format table|json] [--config path]";
7
7
  readonly rebuild: "rebuild [--config path]";
8
- readonly query: "query \"<sql>\" [params...] [--format table|json] [--config path]";
9
- readonly search: "search \"<terms>\" [--preset name] [--include glob ...] [--where \"<sql>\"] [--k n] [--lexical] [--format table|json] [--config path]";
8
+ readonly download: "download [--config path]";
9
+ readonly sql: "sql \"<statement>\" [params...] [--format table|json] [--config path]";
10
+ readonly search: "search \"<terms>\" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json] [--config path]";
10
11
  readonly map: "map [--format table|json] [--config path]";
11
- readonly peek: "peek <path> [--format table|json] [--config path]";
12
+ readonly peek: "peek <path> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--format table|json] [--config path]";
13
+ readonly path: "path <a> <b> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--max-depth n] [--format table|json] [--config path]";
14
+ readonly related: "related <note> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json] [--config path]";
12
15
  };
13
16
  export declare const COMMANDS: Record<string, () => Promise<{
14
17
  default: Command;