sensemaking 0.9.4 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -100
- package/dist/cjs/cli/check.js +7 -13
- package/dist/cjs/cli/check.js.map +1 -1
- package/dist/cjs/cli/download.d.cts +3 -0
- package/dist/cjs/cli/download.d.ts +3 -0
- package/dist/cjs/cli/download.js +214 -0
- package/dist/cjs/cli/download.js.map +1 -0
- package/dist/cjs/cli/index.d.cts +6 -3
- package/dist/cjs/cli/index.d.ts +6 -3
- package/dist/cjs/cli/index.js +23 -5
- package/dist/cjs/cli/index.js.map +1 -1
- package/dist/cjs/cli/init.js +4 -1
- package/dist/cjs/cli/init.js.map +1 -1
- package/dist/cjs/cli/map.js +2 -2
- package/dist/cjs/cli/map.js.map +1 -1
- package/dist/cjs/cli/named.js +10 -18
- package/dist/cjs/cli/named.js.map +1 -1
- package/dist/cjs/cli/path.d.cts +3 -0
- package/dist/cjs/cli/path.d.ts +3 -0
- package/dist/cjs/cli/path.js +161 -0
- package/dist/cjs/cli/path.js.map +1 -0
- package/dist/cjs/cli/peek.js +3 -2
- package/dist/cjs/cli/peek.js.map +1 -1
- package/dist/cjs/cli/related.d.cts +3 -0
- package/dist/cjs/cli/related.d.ts +3 -0
- package/dist/cjs/cli/related.js +279 -0
- package/dist/cjs/cli/related.js.map +1 -0
- package/dist/cjs/cli/search.js +3 -7
- package/dist/cjs/cli/search.js.map +1 -1
- package/dist/cjs/cli/shared.d.cts +3 -1
- package/dist/cjs/cli/shared.d.ts +3 -1
- package/dist/cjs/cli/shared.js +28 -5
- package/dist/cjs/cli/shared.js.map +1 -1
- package/dist/cjs/cli/sql.d.cts +3 -0
- package/dist/cjs/cli/sql.d.ts +3 -0
- package/dist/cjs/cli/{query.js → sql.js} +4 -4
- package/dist/cjs/cli/sql.js.map +1 -0
- package/dist/cjs/cli/status.js +9 -2
- package/dist/cjs/cli/status.js.map +1 -1
- package/dist/cjs/cli.js +14 -15
- package/dist/cjs/cli.js.map +1 -1
- package/dist/cjs/commands.d.cts +12 -4
- package/dist/cjs/commands.d.ts +12 -4
- package/dist/cjs/commands.js +138 -58
- package/dist/cjs/commands.js.map +1 -1
- package/dist/cjs/config.d.cts +6 -4
- package/dist/cjs/config.d.ts +6 -4
- package/dist/cjs/config.js +137 -58
- package/dist/cjs/config.js.map +1 -1
- package/dist/cjs/db.js +14 -34
- package/dist/cjs/db.js.map +1 -1
- package/dist/cjs/errors.d.cts +1 -1
- package/dist/cjs/errors.d.ts +1 -1
- package/dist/cjs/errors.js.map +1 -1
- package/dist/cjs/features/embed.d.cts +14 -0
- package/dist/cjs/features/embed.d.ts +14 -0
- package/dist/cjs/features/embed.js +248 -54
- package/dist/cjs/features/embed.js.map +1 -1
- package/dist/cjs/features/links.js +7 -16
- package/dist/cjs/features/links.js.map +1 -1
- package/dist/cjs/features/rank.js +2 -6
- package/dist/cjs/features/rank.js.map +1 -1
- package/dist/cjs/index.d.cts +1 -1
- package/dist/cjs/index.d.ts +1 -1
- package/dist/cjs/index.js.map +1 -1
- package/dist/cjs/output.js +11 -14
- package/dist/cjs/output.js.map +1 -1
- package/dist/cjs/progress.js +2 -4
- package/dist/cjs/progress.js.map +1 -1
- package/dist/cjs/scan.d.cts +1 -0
- package/dist/cjs/scan.d.ts +1 -0
- package/dist/cjs/scan.js +67 -32
- package/dist/cjs/scan.js.map +1 -1
- package/dist/cjs/search-error.js +4 -6
- package/dist/cjs/search-error.js.map +1 -1
- package/dist/cjs/traverse.d.cts +8 -0
- package/dist/cjs/traverse.d.ts +8 -0
- package/dist/cjs/traverse.js +97 -0
- package/dist/cjs/traverse.js.map +1 -0
- package/dist/esm/cli/check.js +7 -13
- package/dist/esm/cli/check.js.map +1 -1
- package/dist/esm/cli/download.d.ts +3 -0
- package/dist/esm/cli/download.js +27 -0
- package/dist/esm/cli/download.js.map +1 -0
- package/dist/esm/cli/index.d.ts +6 -3
- package/dist/esm/cli/index.js +13 -7
- package/dist/esm/cli/index.js.map +1 -1
- package/dist/esm/cli/init.js +4 -1
- package/dist/esm/cli/init.js.map +1 -1
- package/dist/esm/cli/map.js +3 -2
- package/dist/esm/cli/map.js.map +1 -1
- package/dist/esm/cli/named.js +10 -16
- package/dist/esm/cli/named.js.map +1 -1
- package/dist/esm/cli/path.d.ts +3 -0
- package/dist/esm/cli/path.js +51 -0
- package/dist/esm/cli/path.js.map +1 -0
- package/dist/esm/cli/peek.js +4 -2
- package/dist/esm/cli/peek.js.map +1 -1
- package/dist/esm/cli/related.d.ts +3 -0
- package/dist/esm/cli/related.js +26 -0
- package/dist/esm/cli/related.js.map +1 -0
- package/dist/esm/cli/search.js +2 -5
- package/dist/esm/cli/search.js.map +1 -1
- package/dist/esm/cli/shared.d.ts +3 -1
- package/dist/esm/cli/shared.js +28 -5
- package/dist/esm/cli/shared.js.map +1 -1
- package/dist/esm/cli/sql.d.ts +3 -0
- package/dist/esm/cli/{query.js → sql.js} +4 -4
- package/dist/esm/cli/sql.js.map +1 -0
- package/dist/esm/cli/status.js +8 -1
- package/dist/esm/cli/status.js.map +1 -1
- package/dist/esm/cli.js +13 -10
- package/dist/esm/cli.js.map +1 -1
- package/dist/esm/commands.d.ts +12 -4
- package/dist/esm/commands.js +111 -68
- package/dist/esm/commands.js.map +1 -1
- package/dist/esm/config.d.ts +6 -4
- package/dist/esm/config.js +126 -75
- package/dist/esm/config.js.map +1 -1
- package/dist/esm/db.js +16 -38
- package/dist/esm/db.js.map +1 -1
- package/dist/esm/errors.d.ts +1 -1
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/features/embed.d.ts +14 -0
- package/dist/esm/features/embed.js +110 -35
- package/dist/esm/features/embed.js.map +1 -1
- package/dist/esm/features/links.js +8 -17
- package/dist/esm/features/links.js.map +1 -1
- package/dist/esm/features/rank.js +2 -6
- package/dist/esm/features/rank.js.map +1 -1
- package/dist/esm/features/types.js.map +1 -1
- package/dist/esm/index.d.ts +1 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/output.js +13 -16
- package/dist/esm/output.js.map +1 -1
- package/dist/esm/progress.js +2 -4
- package/dist/esm/progress.js.map +1 -1
- package/dist/esm/scan.d.ts +1 -0
- package/dist/esm/scan.js +41 -34
- package/dist/esm/scan.js.map +1 -1
- package/dist/esm/search-error.js +4 -6
- package/dist/esm/search-error.js.map +1 -1
- package/dist/esm/traverse.d.ts +8 -0
- package/dist/esm/traverse.js +71 -0
- package/dist/esm/traverse.js.map +1 -0
- package/package.json +18 -5
- package/schema.json +12 -16
- package/skills/sense/EXAMPLES.md +13 -22
- package/skills/sense/SKILL.md +65 -140
- package/skills/sense-setup/EXAMPLES.md +30 -46
- package/skills/sense-setup/SKILL.md +22 -83
- package/dist/cjs/cli/query.d.cts +0 -3
- package/dist/cjs/cli/query.d.ts +0 -3
- package/dist/cjs/cli/query.js.map +0 -1
- package/dist/esm/cli/query.d.ts +0 -3
- package/dist/esm/cli/query.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,34 +1,28 @@
|
|
|
1
1
|
# sensemaking
|
|
2
2
|
|
|
3
|
-
Query and search
|
|
4
|
-
the prose — words, links, and meaning fused. A CLI that starts, answers, and exits — no
|
|
5
|
-
server, no build step.
|
|
3
|
+
Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step.
|
|
6
4
|
|
|
7
5
|
## Problem
|
|
8
6
|
|
|
9
|
-
|
|
10
|
-
means grepping or reading whole folders into context. The structure that makes notes navigable —
|
|
11
|
-
frontmatter, wikilinks, headings — is exactly what an agent needs, but nothing exposes it as a
|
|
12
|
-
query surface.
|
|
7
|
+
Markdown notes accumulate: research, decisions, meeting notes, agent output. Past a few dozen, finding the ones relevant to what you're doing means grepping or reading whole folders into context. The structure that makes notes navigable (frontmatter, wikilinks, headings) is exactly what a query needs, but nothing exposes it as a query surface.
|
|
13
8
|
|
|
14
|
-
`sense` indexes all of it into SQLite and reconciles against file timestamps on every query, so
|
|
15
|
-
results are never stale and nothing has to be running.
|
|
9
|
+
`sense` indexes all of it into SQLite and reconciles against file timestamps on every query, so results are never stale and nothing has to be running.
|
|
16
10
|
|
|
17
11
|
## Quick start
|
|
18
12
|
|
|
19
13
|
```bash
|
|
20
14
|
npm install -g sensemaking
|
|
21
15
|
cd your-notes && sense init
|
|
16
|
+
sense download # the embedding model, once per machine; nothing fetches it implicitly
|
|
22
17
|
```
|
|
23
18
|
|
|
24
|
-
Needs Node 22.
|
|
25
|
-
`sense search` indexes prose with.
|
|
19
|
+
Needs Node 22.20 or newer: the first release whose built-in SQLite has both FTS5, which `sense search` indexes prose with, and row-returning `INSERT ... RETURNING`, which `sense path` and `peek` walk the link graph with.
|
|
26
20
|
|
|
27
21
|
```bash
|
|
28
22
|
sense map # orient: fields, hub notes, recent changes
|
|
29
23
|
sense search "revenue OR earnings" --k 10 # locate: words + links + meaning, one ranked list
|
|
30
24
|
sense peek notes/q3-report.md # structure: outline + links, before reading
|
|
31
|
-
sense
|
|
25
|
+
sense sql "SELECT path FROM frontmatter WHERE has(tags, ?)" urgent
|
|
32
26
|
```
|
|
33
27
|
|
|
34
28
|
## Model
|
|
@@ -42,14 +36,9 @@ Every file becomes rows in these tables, plus whatever an enabled feature adds o
|
|
|
42
36
|
| `links` | `src`, `target` as written, `dst` resolved (`NULL` = dead link) | graph |
|
|
43
37
|
| `sections` | heading, `level`, `start_line`, `end_line`, `tokens` estimate | structure |
|
|
44
38
|
|
|
45
|
-
Results are references
|
|
46
|
-
afterward through the filesystem, scoped to the line ranges `peek` returns. This is the
|
|
47
|
-
[just-in-time context pattern](https://www.anthropic.com/engineering/effective-context-engineering-for-ai-agents):
|
|
48
|
-
the agent holds lightweight identifiers and loads payloads only when needed.
|
|
39
|
+
Results are references (path, title, summary, excerpt), never file contents. Reading happens afterward through the filesystem, scoped to the line ranges `peek` returns. This is the [just-in-time context pattern](https://www.anthropic.com/engineering/effective-context-engineering-for-ai-agents): the agent holds lightweight identifiers and loads payloads only when needed.
|
|
49
40
|
|
|
50
|
-
Output size is a contract, measured per release ([BENCHMARKING.md](BENCHMARKING.md)): `map` is
|
|
51
|
-
fixed-size, a `find` row is tens of tokens, and a `peek` stays flat however large the note is —
|
|
52
|
-
so what it saves over reading grows with the file, and a small note is cheaper read whole.
|
|
41
|
+
Output size is a contract, measured per release ([BENCHMARKING.md](BENCHMARKING.md)): `map` is fixed-size, a `search` row is tens of tokens, and a `peek` stays flat however large the note is. What it saves over reading grows with the file; a small note is cheaper to read whole.
|
|
53
42
|
|
|
54
43
|
```sql
|
|
55
44
|
-- filter and search compose in one query
|
|
@@ -64,88 +53,64 @@ ORDER BY bm25(content, 10.0, 5.0, 1.0) LIMIT 10
|
|
|
64
53
|
| command | does |
|
|
65
54
|
|---|---|
|
|
66
55
|
| `map` | doc count, frontmatter field coverage, top hubs by link rank, recent changes |
|
|
67
|
-
| `search "<text>" [--preset name] [--include glob] [--where "<sql>"] [--k n]
|
|
68
|
-
| `peek <path
|
|
69
|
-
| `
|
|
56
|
+
| `search "<text>" [--preset name] [--include glob] [--exclude glob] [--no-exclude] [--where "<sql>"] [--k n]` | words + links + vectors, one fused ranked list; `via` labels each row's evidence |
|
|
57
|
+
| `peek <path> [--preset name] [--where "<sql>"]` | frontmatter + heading outline (`[L143-162, ~380t]`) + links both ways (first 20 per list, each with its total) |
|
|
58
|
+
| `path <a> <b> [--max-depth n] [--preset name] [--where "<sql>"]` | shortest link chain between two notes, or none within the bound |
|
|
59
|
+
| `related <note> [--k n] [--preset name] [--where "<sql>"]` | notes similar in meaning that `<note>` does not yet link to; reads vectors, so semantic-search cost |
|
|
60
|
+
| `sql "<statement>" [params...]` | ad-hoc SQL over all the tables; `?` binds positional args |
|
|
70
61
|
| `<name> [params...]` | run a query saved in the config; `--list` names them |
|
|
71
62
|
| `init` | write a starter `sense.config.json` |
|
|
72
63
|
| `status` | index location, doc count, per-preset coverage, watcher heartbeat |
|
|
73
|
-
| `check` | run every saved query
|
|
64
|
+
| `check` | run every saved query, so a broken one fails here instead of mid-task |
|
|
74
65
|
| `rebuild` | delete the cache and re-crawl |
|
|
66
|
+
| `download` | fetch the embedding model named in the config (once per machine; nothing else downloads it) |
|
|
75
67
|
| `watch` | keep the index warm in the background (optional; see [WATCH.md](WATCH.md)) |
|
|
76
68
|
|
|
77
|
-
`search` runs one text through every engine its scope has
|
|
78
|
-
bare words AND-join, operators are yours), a personalized-PageRank walk over the link graph,
|
|
79
|
-
and vector similarity — fused into one list. `via` labels each row's evidence (`match`,
|
|
80
|
-
`link`, `vector`, combinations); `similarity` is the cosine against the best-matching chunk;
|
|
81
|
-
`lines` points at the section that earned the row — a direct read range. Rows that only
|
|
82
|
-
vectors produced are the "these words aren't in the tree, this is what's near in meaning"
|
|
83
|
-
signal. `--preset` picks a named settings bundle from the config, `--lexical` skips vectors
|
|
84
|
-
for one command, `--where` filters on frontmatter. `--format json` on any reporting command returns
|
|
85
|
-
structured output; `--version` and `--help` do what they say.
|
|
69
|
+
`search` runs one text through every engine its scope has: FTS5 word match (BM25-ranked, bare words AND-join, operators are yours), a personalized-PageRank walk over the link graph, and vector similarity, fused into one list. `via` labels each row's evidence (`match`, `link`, `vector`, combinations); `similarity` is the cosine against the best-matching chunk; `lines` points at the section that earned the row (a direct read range). A `vector`-only row means the search words don't appear in that note; it showed up because the model judged it semantically related. `--preset` picks a named settings bundle from the config, `--where` filters on frontmatter. `--format json` on any reporting command returns structured output; `--version` and `--help` do what they say.
|
|
86
70
|
|
|
87
71
|
## Config
|
|
88
72
|
|
|
89
|
-
`sense init` writes `sense.config.json`; discovery walks up from cwd like git
|
|
90
|
-
(`--config <path>` overrides). Three keys: **presets** (named, self-contained setting
|
|
91
|
-
bundles), **queries** (saved commands), and the version.
|
|
73
|
+
`sense init` writes `sense.config.json`; discovery walks up from cwd like git (`--config <path>` overrides).
|
|
92
74
|
|
|
93
75
|
```json
|
|
94
76
|
{
|
|
95
77
|
"$schema": "https://unpkg.com/sensemaking/schema.json",
|
|
96
|
-
"version":
|
|
78
|
+
"version": 4,
|
|
97
79
|
"presets": {
|
|
98
80
|
"default": { "include": ["**/*.md"], "k": 10 },
|
|
99
|
-
"raw": { "include": ["raw/**/*.md"], "k": 5
|
|
81
|
+
"raw": { "include": ["raw/**/*.md"], "k": 5 }
|
|
100
82
|
},
|
|
83
|
+
"embed": { "model": "minishlab/potion-retrieval-32M", "type": "static" },
|
|
101
84
|
"queries": {
|
|
102
|
-
"dead-links": "SELECT src, target FROM links WHERE dst IS NULL",
|
|
103
|
-
"by-tag": "SELECT path, title FROM frontmatter WHERE has(tags, ?) ORDER BY path",
|
|
104
|
-
"hot":
|
|
85
|
+
"dead-links": { "sql": "SELECT src, target FROM links WHERE dst IS NULL" },
|
|
86
|
+
"by-tag": { "sql": "SELECT path, title FROM frontmatter WHERE has(tags, ?) ORDER BY path" },
|
|
87
|
+
"hot": { "search": "pricing OR billing", "preset": "raw" }
|
|
105
88
|
}
|
|
106
89
|
}
|
|
107
90
|
```
|
|
108
91
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
`
|
|
112
|
-
the
|
|
113
|
-
|
|
114
|
-
`
|
|
115
|
-
preset caused it.
|
|
92
|
+
| key | holds |
|
|
93
|
+
|---|---|
|
|
94
|
+
| `presets` | named bundles of `include`/`exclude` globs, `k` (result count), `semantic` (vectors for this scope, on unless `false`), `where` (a standing SQL filter). A file is indexed if any preset includes it, embedded if a model is named and some covering preset has `semantic` on; `status` shows each preset's coverage. |
|
|
95
|
+
| `embed` | the model vectors are built with. Naming one gives the tree vectors; omitting the block means none at all, whatever the presets say. `sense download` fetches it. |
|
|
96
|
+
| `queries` | entries runnable as `sense <name>`, each naming the verb it runs: `{ sql }` for SQL (`?` binds positional args) or `{ search }` for a ranked search with its settings baked in, so `sense hot` needs no flags. `sense check` runs them all, so a typo'd column fails at check time instead of mid-task. |
|
|
97
|
+
| `version` | schema version; older configs auto-migrate on load, noted on stderr. |
|
|
116
98
|
|
|
117
|
-
|
|
118
|
-
settings baked in — `sense hot` needs no flags. `sense check` runs them all, so a typo'd
|
|
119
|
-
column fails at check time instead of mid-task. Older config versions auto-migrate on
|
|
120
|
-
load, noted on stderr.
|
|
99
|
+
Bare commands use the `default` preset; `--preset` names another; flags override single fields. Editing a preset rebuilds the cache and says which preset caused it.
|
|
121
100
|
|
|
122
|
-
Vectors
|
|
123
|
-
never in the package); an optional top-level `"embed": { "model", "type", "url", "key" }`
|
|
124
|
-
block points at any Model2Vec model, local path, or OpenAI-compatible endpoint (Ollama,
|
|
125
|
-
LM Studio, hosted). Embedding happens on the first semantic search, with progress.
|
|
101
|
+
Vectors need a model, and nothing downloads it implicitly: `sense download` fetches it once per machine into `$XDG_CACHE_HOME/sensemaking/models` (or `~/.cache/...`), one directory per model, shared by every tree (124 MB, never in the package). `embed.model` is a Hugging Face id, or a path to a directory holding `model.safetensors` and `tokenizer.json`, which `sense download` leaves to you to populate. A preset that asks for vectors without the model is an error naming `sense download`, rather than a quieter result that would make the same search answer differently before and after; a `semantic: false` preset never asks, so it is unaffected. An optional top-level `"embed": { "model", "type", "url", "key" }` block points at any Model2Vec model, local path, or OpenAI-compatible endpoint (Ollama, LM Studio, hosted). Embedding the notes themselves happens on the first semantic search, with progress.
|
|
126
102
|
|
|
127
|
-
`has(field, value)` is the one custom SQL function: array membership on JSON-array fields,
|
|
128
|
-
substring on strings, false on missing keys. Frontmatter parsing is lenient — syntax errors are
|
|
129
|
-
per-file warnings and the values are still indexed, so one bad note never costs you the crawl.
|
|
103
|
+
`has(field, value)` is the one custom SQL function: array membership on JSON-array fields, substring on strings, false on missing keys. Frontmatter parsing is lenient: syntax errors are per-file warnings, and the values are still indexed, so one bad note never costs you the crawl.
|
|
130
104
|
|
|
131
105
|
## Scale
|
|
132
106
|
|
|
133
|
-
Every query starts with a freshness check against the cache in `.sense/`; only changed files are
|
|
134
|
-
re-parsed. What to expect as a tree grows:
|
|
107
|
+
Every query starts with a freshness check against the cache in `.sense/`; only changed files are re-parsed. What to expect as a tree grows:
|
|
135
108
|
|
|
136
|
-
- **Work is linear in note count
|
|
137
|
-
|
|
138
|
-
- **
|
|
139
|
-
one: context cost is bounded by what you ask for, not by how much there is.
|
|
140
|
-
- **Bulk changes are paid by whoever queries next.** `sense watch` moves that re-parse into the
|
|
141
|
-
background ([WATCH.md](WATCH.md)) — it changes latency, never answers, since every query
|
|
142
|
-
reconciles for itself. `sense rebuild` starts the cache over.
|
|
109
|
+
- **Work is linear in note count.** Crawl, reconcile, and the freshness check are what every invocation pays; that check is the floor cost of a query and the first thing to watch on a large tree.
|
|
110
|
+
- **Output is flat.** `map`, `peek`, and a search row cost the same on a small tree as a large one: context cost is bounded by what you ask for, not by how much there is.
|
|
111
|
+
- **Bulk changes are paid by whoever queries next.** `sense watch` moves that re-parse into the background ([WATCH.md](WATCH.md)): it changes latency, never answers, since every query reconciles for itself. `sense rebuild` starts the cache over.
|
|
143
112
|
|
|
144
|
-
These are release gates rather than hopes: every release regenerates the numbers on pinned
|
|
145
|
-
corpora spanning a 4x range in note count plus a stress tree that packs the worst measured
|
|
146
|
-
shapes into one place — a megabyte-scale note, heading-dense outlines, dense link graphs,
|
|
147
|
-
hundreds of frontmatter fields. A row that grows faster than linearly, or a token count
|
|
148
|
-
that grows at all, blocks the release. Current figures: [BENCHMARKING.md](BENCHMARKING.md).
|
|
113
|
+
These are release gates rather than hopes: every release regenerates the numbers on pinned corpora spanning a 4x range in note count plus a stress tree that packs the worst measured shapes into one place: a megabyte-scale note, heading-dense outlines, dense link graphs, hundreds of frontmatter fields. A row that grows faster than linearly, or a token count that grows at all, blocks the release. Current figures: [BENCHMARKING.md](BENCHMARKING.md).
|
|
149
114
|
|
|
150
115
|
## For AI agents
|
|
151
116
|
|
|
@@ -153,40 +118,25 @@ that grows at all, blocks the release. Current figures: [BENCHMARKING.md](BENCHM
|
|
|
153
118
|
npx skills add kmalakoff/sensemaking # -g for global, -a claude-code to target
|
|
154
119
|
```
|
|
155
120
|
|
|
156
|
-
Two skills: `sense` for querying a tree
|
|
157
|
-
`via`/`score`/`similarity` columns, worked examples — and `sense-setup` for making one, where
|
|
158
|
-
features, frontmatter conventions, and note size are decisions with consequences either way.
|
|
121
|
+
Two skills: `sense` for querying a tree (what each command is for, FTS5 syntax, reading the `via`/`score`/`similarity` columns, worked examples) and `sense-setup` for making one, where features, frontmatter conventions, and note size are decisions with consequences either way.
|
|
159
122
|
|
|
160
123
|
## Prior art
|
|
161
124
|
|
|
162
|
-
- [Effective context engineering for AI agents](https://www.anthropic.com/engineering/effective-context-engineering-for-ai-agents)
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
- [llm-wiki](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f) (Karpathy): an
|
|
166
|
-
agent-maintained wiki navigated by an `index.md` and links, which he notes needs real search
|
|
167
|
-
infrastructure past a few hundred pages. `sense map` derives that index from the notes instead of
|
|
168
|
-
maintaining it; `find` is the hybrid local search it calls for.
|
|
169
|
-
- Agent memory patterns — llm-wiki's raw/wiki split, Claude Code's dreaming-style nightly
|
|
170
|
-
consolidation — are trees of small notes with metadata, links, and layers of differing
|
|
171
|
-
authority. sense is the query layer such patterns need (filter by metadata and age, scope
|
|
172
|
-
by layer, surface near-duplicates semantically), not an implementation of any one of them.
|
|
125
|
+
- [Effective context engineering for AI agents](https://www.anthropic.com/engineering/effective-context-engineering-for-ai-agents) (Anthropic): agents should hold lightweight identifiers (file paths, links) and load payloads just in time, because context is a finite resource. The commands implement that pattern as a CLI.
|
|
126
|
+
- [llm-wiki](https://gist.github.com/karpathy/442a6bf555914893e9891c11519de94f) (Karpathy): an agent-maintained wiki navigated by an `index.md` and links, which he notes needs real search infrastructure past a few hundred pages. `sense map` derives that index from the notes instead of maintaining it; `sense search` is the hybrid local search it calls for.
|
|
127
|
+
- Agent memory patterns (llm-wiki's raw/wiki split, Claude Code's dreaming-style nightly consolidation) are trees of small notes with metadata, links, and layers of differing authority. sense is the query layer such patterns need: filter by metadata and age, scope by layer, surface near-duplicates semantically. It isn't an implementation of any one of them.
|
|
173
128
|
|
|
174
129
|
## Alternatives
|
|
175
130
|
|
|
176
|
-
- **Obsidian Bases/Dataview
|
|
177
|
-
|
|
178
|
-
- **
|
|
179
|
-
- **
|
|
180
|
-
- **
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
Dependencies: [yaml](https://github.com/eemeli/yaml),
|
|
186
|
-
[remove-markdown](https://github.com/zuchka/remove-markdown),
|
|
187
|
-
[fast-glob](https://github.com/mrmlnc/fast-glob),
|
|
188
|
-
[@huggingface/tokenizers](https://github.com/huggingface/tokenizers.js) (pure JS), and
|
|
189
|
-
Node's built-in SQLite. No native builds.
|
|
131
|
+
- **Obsidian Bases/Dataview:** same filters, but only inside the running app; agents can't query it headless.
|
|
132
|
+
- **Index-on-build tools (MarkdownDB):** query a snapshot; `sense` reconciles on every query.
|
|
133
|
+
- **Note CLIs (zk):** fixed schema; `sense` filters on arbitrary frontmatter.
|
|
134
|
+
- **Graph/LSP tools (IWE):** structural queries over a markdown graph via LSP/CLI/MCP, retrieval by structure rather than similarity; no SQL, no vector search.
|
|
135
|
+
- **Markdown vector stores (markdown-vdb):** hybrid BM25 + vector search over markdown files, no frontmatter filtering; `sense` treats vectors as one signal alongside SQL, not the whole store.
|
|
136
|
+
- **RAG / vector stores:** similarity can't express `WHERE status = 'active'`. Here vectors are one signal inside `search`: same SQLite file, filters compose, every row labels its evidence (`via`), and a preset turns vectors off per layer of the tree. No second store, no daemon, no native builds.
|
|
137
|
+
- **Document-OS apps (Anytype, Logseq, SilverBullet, Capacities):** full applications with their own UI and storage. `sense` is headless: your files stay files, there's no app to run.
|
|
138
|
+
|
|
139
|
+
Dependencies: [yaml](https://github.com/eemeli/yaml), [remove-markdown](https://github.com/zuchka/remove-markdown), [@huggingface/tokenizers](https://github.com/huggingface/tokenizers.js) (pure JS), and Node's built-in SQLite. No native builds.
|
|
190
140
|
|
|
191
141
|
## License
|
|
192
142
|
|
package/dist/cjs/cli/check.js
CHANGED
|
@@ -219,16 +219,9 @@ function _ts_generator(thisArg, body) {
|
|
|
219
219
|
};
|
|
220
220
|
}
|
|
221
221
|
}
|
|
222
|
-
// A
|
|
223
|
-
//
|
|
224
|
-
//
|
|
225
|
-
// unknown-column errors without executing it; queries that take no parameters are also run
|
|
226
|
-
// for a row count. Parameterised queries are validated but not counted: inventing arguments
|
|
227
|
-
// would report a count for a query nobody ran. Saved searches run lexically with k=1 -- that
|
|
228
|
-
// validates the `where` fragment, the FTS5 terms, and the `preset` name (resolveSearch throws
|
|
229
|
-
// on an unknown one, the same breakage class prepare() catches for SQL strings) -- but never
|
|
230
|
-
// semantically (a semantic pass costs model time and, on api trees, network). Whether a
|
|
231
|
-
// result being empty is good or bad is the reader's judgment -- there is no assertion path.
|
|
222
|
+
// A broken query and an empty one look identical, so prepare each to catch syntax and column
|
|
223
|
+
// errors. Parameterised ones are not run (inventing arguments would report a bogus count);
|
|
224
|
+
// searches probe at k=1 without vectors. Empty results are the reader's judgment, not a fail.
|
|
232
225
|
var check = function check(ctx) {
|
|
233
226
|
return _async_to_generator(function() {
|
|
234
227
|
var _cfg_queries, values, format, cfg, _open, db, warnings, rows, failed, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _step_value, name, entry, _sql_match, count, err, sql, params, statement, count1, err1;
|
|
@@ -278,7 +271,8 @@ var check = function check(ctx) {
|
|
|
278
271
|
where: entry.where,
|
|
279
272
|
preset: entry.preset,
|
|
280
273
|
include: entry.include,
|
|
281
|
-
|
|
274
|
+
exclude: entry.exclude,
|
|
275
|
+
probe: true
|
|
282
276
|
})
|
|
283
277
|
];
|
|
284
278
|
case 4:
|
|
@@ -287,7 +281,7 @@ var check = function check(ctx) {
|
|
|
287
281
|
query: name,
|
|
288
282
|
params: '—',
|
|
289
283
|
rows: '—',
|
|
290
|
-
status: count === 0 ? 'ok, but matches 0 notes (
|
|
284
|
+
status: count === 0 ? 'ok, but matches 0 notes (words and links only)' : 'ok (probe run with k=1)'
|
|
291
285
|
});
|
|
292
286
|
return [
|
|
293
287
|
3,
|
|
@@ -312,7 +306,7 @@ var check = function check(ctx) {
|
|
|
312
306
|
8
|
|
313
307
|
];
|
|
314
308
|
case 7:
|
|
315
|
-
sql =
|
|
309
|
+
sql = entry.sql;
|
|
316
310
|
params = ((_sql_match = sql.match(/\?/g)) !== null && _sql_match !== void 0 ? _sql_match : []).length;
|
|
317
311
|
try {
|
|
318
312
|
statement = db.prepare(sql);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/check.ts"],"sourcesContent":["import { search } from '../commands.ts';\nimport { open } from '../db.ts';\nimport type { Row } from '../output.ts';\nimport { printRows } from '../output.ts';\nimport { USAGE } from './index.ts';\nimport { CONFIG, FORMAT, formatOf, parse, printWarnings } from './shared.ts';\nimport type { Command } from './types.ts';\n\n// A
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/check.ts"],"sourcesContent":["import { search } from '../commands.ts';\nimport { open } from '../db.ts';\nimport type { Row } from '../output.ts';\nimport { printRows } from '../output.ts';\nimport { USAGE } from './index.ts';\nimport { CONFIG, FORMAT, formatOf, parse, printWarnings } from './shared.ts';\nimport type { Command } from './types.ts';\n\n// A broken query and an empty one look identical, so prepare each to catch syntax and column\n// errors. Parameterised ones are not run (inventing arguments would report a bogus count);\n// searches probe at k=1 without vectors. Empty results are the reader's judgment, not a fail.\nconst check: Command = async (ctx) => {\n const { values } = parse(ctx.argv, `usage: ${ctx.name} ${USAGE.check}`, { ...FORMAT, ...CONFIG });\n const format = formatOf(values);\n const cfg = ctx.resolveConfig(values.config as string | undefined);\n const { db, warnings } = open(cfg);\n printWarnings(warnings);\n\n const rows: Row[] = [];\n let failed = 0;\n for (const [name, entry] of Object.entries(cfg.queries ?? {})) {\n if (typeof entry === 'object' && entry !== null && 'search' in entry) {\n try {\n const count = (await search(db, cfg, entry.search, { k: 1, where: entry.where, preset: entry.preset, include: entry.include, exclude: entry.exclude, probe: true })).length;\n rows.push({ query: name, params: '—', rows: '—', status: count === 0 ? 'ok, but matches 0 notes (words and links only)' : 'ok (probe run with k=1)' });\n } catch (err) {\n failed++;\n rows.push({ query: name, params: '—', rows: '—', status: `FAILED: ${(err as Error).message}` });\n }\n continue;\n }\n const sql = entry.sql;\n const params = (sql.match(/\\?/g) ?? []).length;\n try {\n const statement = db.prepare(sql);\n if (params > 0) {\n rows.push({ query: name, params, rows: '—', status: 'ok (not run: needs parameters)' });\n continue;\n }\n const count = (statement.all() as unknown[]).length;\n rows.push({ query: name, params, rows: count, status: count === 0 ? 'ok, but returns 0 rows' : 'ok' });\n } catch (err) {\n failed++;\n rows.push({ query: name, params, rows: '—', status: `FAILED: ${(err as Error).message}` });\n }\n }\n\n db.close();\n if (rows.length === 0) {\n console.log('no saved queries in config');\n return;\n }\n printRows(rows, format);\n if (failed > 0) process.exit(1);\n};\nexport default check;\n"],"names":["check","ctx","cfg","values","format","open","db","warnings","rows","failed","name","entry","sql","count","err","params","statement","parse","argv","USAGE","FORMAT","CONFIG","formatOf","resolveConfig","config","printWarnings","Object","entries","queries","search","k","where","preset","include","exclude","probe","length","push","query","status","message","match","prepare","all","close","console","log","printRows","process","exit"],"mappings":";;;;+BAuDA;;;eAAA;;;0BAvDuB;oBACF;wBAEK;uBACJ;wBACyC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG/D,6FAA6F;AAC7F,2FAA2F;AAC3F,8FAA8F;AAC9F,IAAMA,QAAiB,eAAOC;;YASeC,cARnCC,QACFC,QACAF,KACmBG,OAAjBC,IAAIC,UAGNC,MACFC,QACC,2BAAA,mBAAA,gBAAA,WAAA,oBAAOC,MAAMC,OAYAC,YATNC,OAECC,KAMLF,KACAG,QAEEC,WAKAH;;;;oBA3BFV,SAAWc,IAAAA,eAAK,EAAChB,IAAIiB,IAAI,EAAE,AAAC,UAAqBC,OAAZlB,IAAIS,IAAI,EAAC,KAAe,OAAZS,cAAK,CAACnB,KAAK,GAAI,mBAAKoB,gBAAM,EAAKC,gBAAM,GAAtFlB;oBACFC,SAASkB,IAAAA,kBAAQ,EAACnB;oBAClBD,MAAMD,IAAIsB,aAAa,CAACpB,OAAOqB,MAAM;oBAClBnB,QAAAA,IAAAA,UAAI,EAACH,MAAtBI,KAAiBD,MAAjBC,IAAIC,WAAaF,MAAbE;oBACZkB,IAAAA,uBAAa,EAAClB;oBAERC;oBACFC,SAAS;oBACR,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAuBiB,OAAOC,OAAO,EAACzB,eAAAA,IAAI0B,OAAO,cAAX1B,0BAAAA,eAAe,CAAC;;;2BAAtD,6BAAA,QAAA;;;;mDAAA,iBAAOQ,uBAAMC;yBACZ,CAAA,CAAA,OAAOA,sCAAP,SAAOA,MAAI,MAAM,YAAYA,UAAU,QAAQ,YAAYA,KAAI,GAA/D;;;;;;;;;;;;oBAEe;;wBAAMkB,IAAAA,kBAAM,EAACvB,IAAIJ,KAAKS,MAAMkB,MAAM,EAAE;4BAAEC,GAAG;4BAAGC,OAAOpB,MAAMoB,KAAK;4BAAEC,QAAQrB,MAAMqB,MAAM;4BAAEC,SAAStB,MAAMsB,OAAO;4BAAEC,SAASvB,MAAMuB,OAAO;4BAAEC,OAAO;wBAAK;;;oBAA3JtB,QAAQ,AAAC,cAAsJuB,MAAM;oBAC3K5B,KAAK6B,IAAI,CAAC;wBAAEC,OAAO5B;wBAAMK,QAAQ;wBAAKP,MAAM;wBAAK+B,QAAQ1B,UAAU,IAAI,mDAAmD;oBAA0B;;;;;;oBAC7IC;oBACPL;oBACAD,KAAK6B,IAAI,CAAC;wBAAEC,OAAO5B;wBAAMK,QAAQ;wBAAKP,MAAM;wBAAK+B,QAAQ,AAAC,WAAiC,OAAvB,AAACzB,IAAc0B,OAAO;oBAAG;;;;;;oBAE/F;;;;;oBAEI5B,MAAMD,MAAMC,GAAG;oBACfG,SAAS,EAACH,aAAAA,IAAI6B,KAAK,CAAC,oBAAV7B,wBAAAA,iBAAwBwB,MAAM;oBAC9C,IAAI;wBACIpB,YAAYV,GAAGoC,OAAO,CAAC9B;wBAC7B,IAAIG,SAAS,GAAG;4BACdP,KAAK6B,IAAI,CAAC;gCAAEC,OAAO5B;gCAAMK,QAAAA;gCAAQP,MAAM;gCAAK+B,QAAQ;4BAAiC;4BACrF;;;;wBACF;wBACM1B,SAAQ,AAACG,UAAU2B,GAAG,GAAiBP,MAAM;wBACnD5B,KAAK6B,IAAI,CAAC;4BAAEC,OAAO5B;4BAAMK,QAAAA;4BAAQP,MAAMK;4BAAO0B,QAAQ1B,WAAU,IAAI,2BAA2B;wBAAK;oBACtG,EAAE,OAAOC,KAAK;wBACZL;wBACAD,KAAK6B,IAAI,CAAC;4BAAEC,OAAO5B;4BAAMK,QAAAA;4BAAQP,MAAM;4BAAK+B,QAAQ,AAAC,WAAiC,OAAvB,AAACzB,IAAc0B,OAAO;wBAAG;oBAC1F;;;oBAxBG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBA2BLlC,GAAGsC,KAAK;oBACR,IAAIpC,KAAK4B,MAAM,KAAK,GAAG;wBACrBS,QAAQC,GAAG,CAAC;wBACZ;;;oBACF;oBACAC,IAAAA,mBAAS,EAACvC,MAAMJ;oBAChB,IAAIK,SAAS,GAAGuC,QAAQC,IAAI,CAAC;;;;;;IAC/B;;IACA,WAAejD"}
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", {
|
|
3
|
+
value: true
|
|
4
|
+
});
|
|
5
|
+
Object.defineProperty(exports, "default", {
|
|
6
|
+
enumerable: true,
|
|
7
|
+
get: function() {
|
|
8
|
+
return _default;
|
|
9
|
+
}
|
|
10
|
+
});
|
|
11
|
+
var _configts = require("../config.js");
|
|
12
|
+
var _errorsts = require("../errors.js");
|
|
13
|
+
var _embedts = require("../features/embed.js");
|
|
14
|
+
var _indexts = require("./index.js");
|
|
15
|
+
var _sharedts = require("./shared.js");
|
|
16
|
+
function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
|
|
17
|
+
try {
|
|
18
|
+
var info = gen[key](arg);
|
|
19
|
+
var value = info.value;
|
|
20
|
+
} catch (error) {
|
|
21
|
+
reject(error);
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
if (info.done) {
|
|
25
|
+
resolve(value);
|
|
26
|
+
} else {
|
|
27
|
+
Promise.resolve(value).then(_next, _throw);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
function _async_to_generator(fn) {
|
|
31
|
+
return function() {
|
|
32
|
+
var self = this, args = arguments;
|
|
33
|
+
return new Promise(function(resolve, reject) {
|
|
34
|
+
var gen = fn.apply(self, args);
|
|
35
|
+
function _next(value) {
|
|
36
|
+
asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
|
|
37
|
+
}
|
|
38
|
+
function _throw(err) {
|
|
39
|
+
asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
|
|
40
|
+
}
|
|
41
|
+
_next(undefined);
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
function _define_property(obj, key, value) {
|
|
46
|
+
if (key in obj) {
|
|
47
|
+
Object.defineProperty(obj, key, {
|
|
48
|
+
value: value,
|
|
49
|
+
enumerable: true,
|
|
50
|
+
configurable: true,
|
|
51
|
+
writable: true
|
|
52
|
+
});
|
|
53
|
+
} else {
|
|
54
|
+
obj[key] = value;
|
|
55
|
+
}
|
|
56
|
+
return obj;
|
|
57
|
+
}
|
|
58
|
+
function _object_spread(target) {
|
|
59
|
+
for(var i = 1; i < arguments.length; i++){
|
|
60
|
+
var source = arguments[i] != null ? arguments[i] : {};
|
|
61
|
+
var ownKeys = Object.keys(source);
|
|
62
|
+
if (typeof Object.getOwnPropertySymbols === "function") {
|
|
63
|
+
ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
|
|
64
|
+
return Object.getOwnPropertyDescriptor(source, sym).enumerable;
|
|
65
|
+
}));
|
|
66
|
+
}
|
|
67
|
+
ownKeys.forEach(function(key) {
|
|
68
|
+
_define_property(target, key, source[key]);
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
return target;
|
|
72
|
+
}
|
|
73
|
+
function _ts_generator(thisArg, body) {
|
|
74
|
+
var f, y, t, _ = {
|
|
75
|
+
label: 0,
|
|
76
|
+
sent: function() {
|
|
77
|
+
if (t[0] & 1) throw t[1];
|
|
78
|
+
return t[1];
|
|
79
|
+
},
|
|
80
|
+
trys: [],
|
|
81
|
+
ops: []
|
|
82
|
+
}, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
|
|
83
|
+
return d(g, "next", {
|
|
84
|
+
value: verb(0)
|
|
85
|
+
}), d(g, "throw", {
|
|
86
|
+
value: verb(1)
|
|
87
|
+
}), d(g, "return", {
|
|
88
|
+
value: verb(2)
|
|
89
|
+
}), typeof Symbol === "function" && d(g, Symbol.iterator, {
|
|
90
|
+
value: function() {
|
|
91
|
+
return this;
|
|
92
|
+
}
|
|
93
|
+
}), g;
|
|
94
|
+
function verb(n) {
|
|
95
|
+
return function(v) {
|
|
96
|
+
return step([
|
|
97
|
+
n,
|
|
98
|
+
v
|
|
99
|
+
]);
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
function step(op) {
|
|
103
|
+
if (f) throw new TypeError("Generator is already executing.");
|
|
104
|
+
while(g && (g = 0, op[0] && (_ = 0)), _)try {
|
|
105
|
+
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
106
|
+
if (y = 0, t) op = [
|
|
107
|
+
op[0] & 2,
|
|
108
|
+
t.value
|
|
109
|
+
];
|
|
110
|
+
switch(op[0]){
|
|
111
|
+
case 0:
|
|
112
|
+
case 1:
|
|
113
|
+
t = op;
|
|
114
|
+
break;
|
|
115
|
+
case 4:
|
|
116
|
+
_.label++;
|
|
117
|
+
return {
|
|
118
|
+
value: op[1],
|
|
119
|
+
done: false
|
|
120
|
+
};
|
|
121
|
+
case 5:
|
|
122
|
+
_.label++;
|
|
123
|
+
y = op[1];
|
|
124
|
+
op = [
|
|
125
|
+
0
|
|
126
|
+
];
|
|
127
|
+
continue;
|
|
128
|
+
case 7:
|
|
129
|
+
op = _.ops.pop();
|
|
130
|
+
_.trys.pop();
|
|
131
|
+
continue;
|
|
132
|
+
default:
|
|
133
|
+
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
|
|
134
|
+
_ = 0;
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
|
|
138
|
+
_.label = op[1];
|
|
139
|
+
break;
|
|
140
|
+
}
|
|
141
|
+
if (op[0] === 6 && _.label < t[1]) {
|
|
142
|
+
_.label = t[1];
|
|
143
|
+
t = op;
|
|
144
|
+
break;
|
|
145
|
+
}
|
|
146
|
+
if (t && _.label < t[2]) {
|
|
147
|
+
_.label = t[2];
|
|
148
|
+
_.ops.push(op);
|
|
149
|
+
break;
|
|
150
|
+
}
|
|
151
|
+
if (t[2]) _.ops.pop();
|
|
152
|
+
_.trys.pop();
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
op = body.call(thisArg, _);
|
|
156
|
+
} catch (e) {
|
|
157
|
+
op = [
|
|
158
|
+
6,
|
|
159
|
+
e
|
|
160
|
+
];
|
|
161
|
+
y = 0;
|
|
162
|
+
} finally{
|
|
163
|
+
f = t = 0;
|
|
164
|
+
}
|
|
165
|
+
if (op[0] & 5) throw op[1];
|
|
166
|
+
return {
|
|
167
|
+
value: op[0] ? op[1] : void 0,
|
|
168
|
+
done: true
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
// The one command that fetches weights. Nothing else touches the network for them, so a
|
|
173
|
+
// query that reads like a query never spends minutes pulling 124 MB. Asks nothing, so there
|
|
174
|
+
// is no prompt for CI to bypass; running it twice is a no-op.
|
|
175
|
+
var download = function download(ctx) {
|
|
176
|
+
return _async_to_generator(function() {
|
|
177
|
+
var values, cfg, e, dir;
|
|
178
|
+
return _ts_generator(this, function(_state) {
|
|
179
|
+
switch(_state.label){
|
|
180
|
+
case 0:
|
|
181
|
+
values = (0, _sharedts.parse)(ctx.argv, "usage: ".concat(ctx.name, " ").concat(_indexts.USAGE.download), _object_spread({}, _sharedts.CONFIG)).values;
|
|
182
|
+
cfg = ctx.resolveConfig(values.config);
|
|
183
|
+
e = (0, _configts.embedConfig)(cfg);
|
|
184
|
+
if (!e) throw new _errorsts.SenseError('EMBED_DISABLED', 'no preset asks for vectors, so there is no model to download');
|
|
185
|
+
if (e.type === 'api') {
|
|
186
|
+
console.log('embed type is "api" ('.concat(e.url, "), so there is nothing to download; ").concat(ctx.name, " search reaches it over the network"));
|
|
187
|
+
return [
|
|
188
|
+
2
|
|
189
|
+
];
|
|
190
|
+
}
|
|
191
|
+
if ((0, _embedts.modelPresent)(cfg)) {
|
|
192
|
+
console.log("".concat(e.model, " is already available in ").concat((0, _embedts.modelDir)(e.model)));
|
|
193
|
+
return [
|
|
194
|
+
2
|
|
195
|
+
];
|
|
196
|
+
}
|
|
197
|
+
return [
|
|
198
|
+
4,
|
|
199
|
+
(0, _embedts.downloadModel)(e.model, function(file, into) {
|
|
200
|
+
return console.error("fetching ".concat(e.model, "/").concat(file, " into ").concat(into));
|
|
201
|
+
})
|
|
202
|
+
];
|
|
203
|
+
case 1:
|
|
204
|
+
dir = _state.sent();
|
|
205
|
+
console.log("downloaded ".concat(e.model, " into ").concat(dir));
|
|
206
|
+
return [
|
|
207
|
+
2
|
|
208
|
+
];
|
|
209
|
+
}
|
|
210
|
+
});
|
|
211
|
+
})();
|
|
212
|
+
};
|
|
213
|
+
var _default = download;
|
|
214
|
+
/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/download.ts"],"sourcesContent":["import { embedConfig } from '../config.ts';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, modelDir, modelPresent } from '../features/embed.ts';\nimport { USAGE } from './index.ts';\nimport { CONFIG, parse } from './shared.ts';\nimport type { Command } from './types.ts';\n\n// The one command that fetches weights. Nothing else touches the network for them, so a\n// query that reads like a query never spends minutes pulling 124 MB. Asks nothing, so there\n// is no prompt for CI to bypass; running it twice is a no-op.\nconst download: Command = async (ctx) => {\n const { values } = parse(ctx.argv, `usage: ${ctx.name} ${USAGE.download}`, { ...CONFIG });\n const cfg = ctx.resolveConfig(values.config as string | undefined);\n const e = embedConfig(cfg);\n if (!e) throw new SenseError('EMBED_DISABLED', 'no preset asks for vectors, so there is no model to download');\n\n if (e.type === 'api') {\n console.log(`embed type is \"api\" (${e.url}), so there is nothing to download; ${ctx.name} search reaches it over the network`);\n return;\n }\n if (modelPresent(cfg)) {\n console.log(`${e.model} is already available in ${modelDir(e.model)}`);\n return;\n }\n const dir = await downloadModel(e.model, (file, into) => console.error(`fetching ${e.model}/${file} into ${into}`));\n console.log(`downloaded ${e.model} into ${dir}`);\n};\nexport default download;\n"],"names":["download","ctx","values","cfg","e","dir","parse","argv","USAGE","name","CONFIG","resolveConfig","config","embedConfig","SenseError","type","console","log","url","modelPresent","modelDir","model","downloadModel","file","into","error"],"mappings":";;;;+BA2BA;;;eAAA;;;wBA3B4B;wBACD;uBAC2B;uBAChC;wBACQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG9B,wFAAwF;AACxF,4FAA4F;AAC5F,8DAA8D;AAC9D,IAAMA,WAAoB,kBAAOC;;YACvBC,QACFC,KACAC,GAWAC;;;;oBAbEH,SAAWI,IAAAA,eAAK,EAACL,IAAIM,IAAI,EAAE,AAAC,UAAqBC,OAAZP,IAAIQ,IAAI,EAAC,KAAkB,OAAfD,cAAK,CAACR,QAAQ,GAAI,mBAAKU,gBAAM,GAA9ER;oBACFC,MAAMF,IAAIU,aAAa,CAACT,OAAOU,MAAM;oBACrCR,IAAIS,IAAAA,qBAAW,EAACV;oBACtB,IAAI,CAACC,GAAG,MAAM,IAAIU,oBAAU,CAAC,kBAAkB;oBAE/C,IAAIV,EAAEW,IAAI,KAAK,OAAO;wBACpBC,QAAQC,GAAG,CAAC,AAAC,wBAAmEhB,OAA5CG,EAAEc,GAAG,EAAC,wCAA+C,OAATjB,IAAIQ,IAAI,EAAC;wBACzF;;;oBACF;oBACA,IAAIU,IAAAA,qBAAY,EAAChB,MAAM;wBACrBa,QAAQC,GAAG,CAAC,AAAC,GAAqCG,OAAnChB,EAAEiB,KAAK,EAAC,6BAA6C,OAAlBD,IAAAA,iBAAQ,EAAChB,EAAEiB,KAAK;wBAClE;;;oBACF;oBACY;;wBAAMC,IAAAA,sBAAa,EAAClB,EAAEiB,KAAK,EAAE,SAACE,MAAMC;mCAASR,QAAQS,KAAK,CAAC,AAAC,YAAsBF,OAAXnB,EAAEiB,KAAK,EAAC,KAAgBG,OAAbD,MAAK,UAAa,OAALC;;;;oBAArGnB,MAAM;oBACZW,QAAQC,GAAG,CAAC,AAAC,cAA6BZ,OAAhBD,EAAEiB,KAAK,EAAC,UAAY,OAAJhB;;;;;;IAC5C;;IACA,WAAeL"}
|
package/dist/cjs/cli/index.d.cts
CHANGED
|
@@ -5,10 +5,13 @@ export declare const USAGE: {
|
|
|
5
5
|
readonly status: "status [--format table|json] [--config path]";
|
|
6
6
|
readonly check: "check [--format table|json] [--config path]";
|
|
7
7
|
readonly rebuild: "rebuild [--config path]";
|
|
8
|
-
readonly
|
|
9
|
-
readonly
|
|
8
|
+
readonly download: "download [--config path]";
|
|
9
|
+
readonly sql: "sql \"<statement>\" [params...] [--format table|json] [--config path]";
|
|
10
|
+
readonly search: "search \"<terms>\" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json] [--config path]";
|
|
10
11
|
readonly map: "map [--format table|json] [--config path]";
|
|
11
|
-
readonly peek: "peek <path> [--format table|json] [--config path]";
|
|
12
|
+
readonly peek: "peek <path> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--format table|json] [--config path]";
|
|
13
|
+
readonly path: "path <a> <b> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--max-depth n] [--format table|json] [--config path]";
|
|
14
|
+
readonly related: "related <note> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json] [--config path]";
|
|
12
15
|
};
|
|
13
16
|
export declare const COMMANDS: Record<string, () => Promise<{
|
|
14
17
|
default: Command;
|
package/dist/cjs/cli/index.d.ts
CHANGED
|
@@ -5,10 +5,13 @@ export declare const USAGE: {
|
|
|
5
5
|
readonly status: "status [--format table|json] [--config path]";
|
|
6
6
|
readonly check: "check [--format table|json] [--config path]";
|
|
7
7
|
readonly rebuild: "rebuild [--config path]";
|
|
8
|
-
readonly
|
|
9
|
-
readonly
|
|
8
|
+
readonly download: "download [--config path]";
|
|
9
|
+
readonly sql: "sql \"<statement>\" [params...] [--format table|json] [--config path]";
|
|
10
|
+
readonly search: "search \"<terms>\" [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json] [--config path]";
|
|
10
11
|
readonly map: "map [--format table|json] [--config path]";
|
|
11
|
-
readonly peek: "peek <path> [--format table|json] [--config path]";
|
|
12
|
+
readonly peek: "peek <path> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--format table|json] [--config path]";
|
|
13
|
+
readonly path: "path <a> <b> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--max-depth n] [--format table|json] [--config path]";
|
|
14
|
+
readonly related: "related <note> [--preset name] [--include glob ...] [--exclude glob ...] [--no-exclude] [--where \"<sql>\"] [--k n] [--format table|json] [--config path]";
|
|
12
15
|
};
|
|
13
16
|
export declare const COMMANDS: Record<string, () => Promise<{
|
|
14
17
|
default: Command;
|