sensemaking 0.8.0 → 0.9.0

package/skills/sense-setup/SKILL.md

@@ -1,82 +1,91 @@
 ---
 name: sense-setup
-description: Set up the sense CLI on a markdown tree and make the tree-design decisions that shape it — sense init, enabling features (links, sections, rank, embed), and the trade-offs of frontmatter conventions, summaries, folder layout, and note size. Use when creating or restructuring a markdown knowledge base, running sense init, editing sense.config.json, enabling semantic search, or deciding how notes should be written for an agent to query later.
+description: Set up the sense CLI on a markdown tree and make the tree-design decisions that shape it — sense init, presets (which files, which settings, vectors on or off), and the trade-offs of frontmatter conventions, summaries, folder layout, and note size. Use when creating or restructuring a markdown knowledge base, running sense init, editing sense.config.json, configuring search scope or vectors, or deciding how notes should be written for an agent to query later.
 ---
 
 # sense: setup and tree design
 
 Querying an existing tree is the `sense` skill. This one covers making a tree:
-installing, configuring features, and the design decisions a tree owner faces.
+installing, writing presets, and the design decisions a tree owner faces.
+Worked configurations for common tree shapes: [EXAMPLES.md](EXAMPLES.md).
 
 ## Setup
 
 - `npm install -g sensemaking`, then `sense init` at the tree root writes
-  `sense.config.json` (all opt-out features on, `embed` off). Config discovery
-  walks up from cwd; `--config <path>` overrides.
-- `scan.include` globs resolve relative to the config file, never the cwd.
-- `sense map` and `sense status` report feature state
-  (`features: links, sections, rank · off: embed (features.embed)`), so the
-  current config is always discoverable from output.
+  `sense.config.json` — two presets (`default`, and `large` showing what a big
+  vault tunes), everything on. Config discovery walks up from cwd;
+  `--config <path>` overrides.
+- Globs resolve relative to the config file, never the cwd.
+- `sense status` and `sense map` show each preset's coverage (files matched,
+  embedded count), so what a config actually indexes is always visible in
+  output. A config edit that changes coverage rebuilds the cache and names the
+  preset that caused it on stderr.
 
-## Features
+## Presets
 
-| feature | powers | cost when on | config key |
-|---|---|---|---|
-| `links` | backlinks, dead-link queries, `find`'s link expansion, `peek`'s link lists | link re-resolution at reconcile | `features.links` (default on) |
-| `sections` | `peek`'s outline, line-range reads, per-section token estimates | heading extraction at parse | `features.sections` (default on) |
-| `rank` | `map`'s hubs, `_rank` in any ORDER BY | PageRank pass at reconcile; requires `links` | `features.rank` (default on) |
-| `embed` | `find --semantic` (meaning-based expansion) | the first semantic query embeds the whole tree (minutes on tens of thousands of notes, progress on stderr); after that, a model load per invocation (measured in BENCHMARKING.md as `semantic_find_ms` minus `find_ms`) | `features.embed` (default off) |
+A preset is a named, self-contained bundle of settings. `default` (required) is
+what bare commands use; every other preset is addressed by name
+(`sense search "..." --preset raw`, or `"preset": "raw"` in a saved search).
+No inheritance: what a preset states is all it does.
 
-- `embed` accepts `true` (built-in static model) or an object: `model` (Hugging
-  Face id or local path), `type` (`static` pure-JS built-in, or `api` for any
-  OpenAI-compatible `/v1/embeddings` endpoint — Ollama, LM Studio, llama.cpp,
-  hosted), `url`, `key` (env var name). A local `model` path is fully offline.
-- The two types differ in what they match, not only in cost: `static` is a
-  context-free distilled model that handles paraphrase and reworded concepts
-  (a query like "delegating without micromanaging" can reach a note that uses
-  neither word); tight near-synonyms and domain jargon ("heart attack" for
-  "myocardial infarction") are where it misses and an `api` transformer model
-  tends to succeed. The gain concentrates where the searcher's vocabulary
-  differs from the notes' — measured in the retrieval eval (BENCHMARKING.md,
-  "Retrieval quality"): on a vocabulary-gap corpus semantic expansion adds
-  recall; where vocabulary overlaps, BM25 plus link expansion already answers
-  most queries and vectors mostly reorder.
-- Enabling `embed` changes no default `find` result — expansion runs only when
-  a query passes `--semantic`. Invoking `--semantic` on a tree without `embed`
-  is an error naming the config key.
-- Toggling any feature rebuilds the cache on the next query (safe, automatic).
-- Disabled features degrade output, visibly: `peek` prints
-  `sections: off (features.sections)` rather than an empty outline.
-- Every query reconciles for itself, so nothing has to be running. On a
-  large tree that changes in bulk — a sync, a generated batch — whoever
-  queries next pays that re-parse; `sense watch` moves it into the
-  background instead. It changes latency, never answers, and the OS
-  supervises it rather than the CLI (WATCH.md ships launchd and systemd
-  units).
+| field | means | default |
+|---|---|---|
+| `include` / `exclude` | which files this preset covers (globs) | required |
+| `k` | how many results a search returns | 10 |
+| `semantic` | vectors for this preset's files and searches | on; only ever written as `false` |
+| `where` | a standing SQL filter on frontmatter | none |
+
+**Indexing derives from presets.** A file is indexed if any preset includes it;
+it is embedded if any covering preset has semantic on. Consequences worth
+designing around:
+
+- A layer of the tree covered only by a `semantic: false` preset (raw sources,
+  archives, generated output) is fully searchable lexically and by SQL but
+  costs no vector work — the main scale lever.
+- Files no preset includes are not indexed at all.
+- Presets may overlap; they are views, not partitions.
+- The first semantic search embeds everything covered (progress on stderr;
+  minutes on tens of thousands of notes, seconds on small trees). Vectors use
+  the built-in static model unless a top-level
+  `"embed": { "model", "type": "static"|"api", "url", "key" }` block points at
+  a Model2Vec model, local path, or OpenAI-compatible endpoint. `static`
+  handles paraphrase and reworded concepts; tight domain jargon ("heart
+  attack" for "myocardial infarction") is where an `api` transformer model
+  tends to do better — measured in BENCHMARKING.md, "Retrieval quality".
+- Global `features` (`links`, `sections`, `rank`) still toggle tree-wide;
+  most trees never touch them.
+
+**Large vaults**: everything except the vector build is measured linear to
+100k notes with no tuning (BENCHMARKING.md). The knobs that matter are `k`
+(more, smaller results — rows carry `lines` section ranges, so agents read
+sections, not files) and `semantic: false` on the layers that don't earn
+vectors.
 
 ## Tree design decisions
 
 These belong to the tree's owner. sense works with any of them and reads no
-instruction files of its own; each choice below only changes what queries can
-do, and every consequence is listed so the choice can be made deliberately.
+instruction files of its own; each choice only changes what queries can do.
 
 - **Frontmatter fields.** Columns are discovered per tree — whatever keys notes
   declare become queryable. Consistent fields across notes make SQL filters
   and named queries possible (`WHERE status = 'active'`). SQLite's compiled
   column limit (2,000; sqlite.org/limits.html) bounds distinct keys per tree —
-  the crawl stops with an error naming the count and the levers, which in
-  practice only a generator writing unbounded keys ever hits. Reserved keys
+  the crawl stops with an error naming the count and the levers. Reserved keys
   (dropped with a warning): `path`, `_mtime`, `_size`, `_rank`, `content`,
   `links`, `sections`. Values keep their YAML type: strings TEXT, whole numbers
   and booleans INTEGER (`true` is 1), fractions REAL, lists and maps JSON text;
   `map` prints the observed type per field.
+- **Presets are path-shaped; frontmatter is state-shaped.** A preset's coverage
+  must be computable from the path alone (it decides indexing, baked into the
+  cache). Volatile state (`status`, `project`, dates) lives in frontmatter and
+  filters at query time (`where`, `has()`, `datetime()`). A state worth
+  different *indexing* (retired memory, superseded sources) is a state worth
+  moving the file — the archive-folder pattern in EXAMPLES.md.
 - **What a note omits is also a filter.** A layer that deliberately carries none
   of the fields the saved views filter on is excluded from all of them without
-  any view naming the layer; only a view filtering solely on a field the layer
-  does share needs an explicit condition (`AND type != 'raw'`). Sparse fields cut
-  both ways: less of the tree filters when you want breadth, and exactly this
-  separation when layers differ in authority — source extractions vs.
-  conclusions, generated output vs. notes.
+  any view naming the layer. Sparse fields cut both ways: less of the tree
+  filters when you want breadth, and exactly this separation when layers
+  differ in authority.
 - **Dates.** `datetime()` comparisons work for dates written as ISO 8601 —
   the only format it parses. A tree that mixes date formats can store them,
   but can't compare them in SQL.
@@ -84,22 +93,18 @@ do, and every consequence is listed so the choice can be made deliberately.
   every result row (often answering a question with no file read) and is a
   weighted search field ranked above body text. The cost is writing and
   maintaining the line as notes change.
-- **Folder shape.** sense is structure-indifferent: globs find the files,
-  paths are queryable text, links resolve by basename at any depth. Folders
-  are for the humans and agents navigating the tree, not for the index —
-  flat and nested trees query identically.
-- **Note size.** Many small notes: precise `find` hits, whole-file reads stay
-  cheap, more links to maintain. Fewer large notes: `sections` and `peek`
-  carry the cost down to line-range reads. Both work; per-section token
-  estimates exist either way.
-- **Recurring questions.** A scenario an agent will run repeatedly can be saved
-  with its settings: a SQL string for filters and reports, or a saved find
-  (`"hot": { "find": "...", "k": 20, "semantic": true }`) for searches — either
-  runs as `sense <name>`, and `sense check` validates both kinds against the
-  real tree, so a broken saved scenario fails at check time, not mid-task.
+- **Folder shape.** Globs find the files, paths are queryable text, links
+  resolve by basename at any depth — but presets make folders meaningful:
+  a folder is the natural unit that gets its own coverage and settings.
+- **Note size.** Many small notes: precise search hits, whole-file reads stay
+  cheap, more links to maintain. Fewer large notes: `sections`, `peek`, and
+  the `lines` column carry the cost down to line-range reads. Both work.
+- **Recurring questions.** Save a scenario an agent will repeat: a SQL string
+  for filters and reports, or a saved search
+  (`"hot": { "search": "...", "preset": "raw", "k": 5 }`). Either runs as
+  `sense <name>`, and `sense check` validates both kinds against the real
+  tree, so a broken saved scenario fails at check time, not mid-task.
 - **Where decisions live.** Choices that should outlive one conversation can
   be recorded in the agent's own instruction or skill files, or in a note in
   the tree itself; a one-off search over an existing corpus needs none of
-  that. Whether to settle a choice with the user or proceed on the corpus as
-  found depends on whether the agent is only querying or also authoring —
-  an authoring agent's choices compound; a querying agent's don't.
+  that.

package/dist/cjs/cli/find.d.cts

@@ -1,3 +0,0 @@
-import type { Command } from './types.js';
-declare const findCmd: Command;
-export default findCmd;

package/dist/cjs/cli/find.d.ts

@@ -1,3 +0,0 @@
-import type { Command } from './types.js';
-declare const findCmd: Command;
-export default findCmd;

package/dist/cjs/cli/find.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/find.ts"],"sourcesContent":["import { find } from '../commands.ts';\nimport { printRows } from '../output.ts';\nimport { parseK, withDb } from './shared.ts';\nimport type { Command } from './types.ts';\n\nconst findCmd: Command = async (ctx) => {\n  const [terms] = ctx.rest;\n  if (!terms) ctx.usageError(`usage: ${ctx.name} find \"<terms>\" [--where \"<sql>\"] [--k n] [--semantic]`);\n  const k = parseK(ctx);\n  await withDb(ctx, async (db, cfg) => printRows(await find(db, cfg, terms, { k, where: ctx.values.where, semantic: ctx.values.semantic }), ctx.format));\n};\nexport default findCmd;\n"],"names":["findCmd","ctx","terms","k","rest","usageError","name","parseK","withDb","db","cfg","find","where","values","semantic","printRows","format"],"mappings":";;;;+BAWA;;;eAAA;;;0BAXqB;wBACK;wBACK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG/B,IAAMA,UAAmB,iBAAOC;;YACdA,WAATC,OAEDC;;;;oBAFUF,6BAAAA,IAAIG,IAAI,MAAjBF,QAASD;oBAChB,IAAI,CAACC,OAAOD,IAAII,UAAU,CAAC,AAAC,UAAkB,OAATJ,IAAIK,IAAI,EAAC;oBACxCH,IAAII,IAAAA,gBAAM,EAACN;oBACjB;;wBAAMO,IAAAA,gBAAM,EAACP,KAAK,SAAOQ,IAAIC;;;;;4CAAkB;;gDAAMC,IAAAA,gBAAI,EAACF,IAAIC,KAAKR,OAAO;oDAAEC,GAAAA;oDAAGS,OAAOX,IAAIY,MAAM,CAACD,KAAK;oDAAEE,UAAUb,IAAIY,MAAM,CAACC,QAAQ;gDAAC;;;;;gDAAjGC,mBAAS;oDAAC;oDAA2Fd,IAAIe,MAAM;;;;;;;;;oBAApJ;;;;;;IACF;;IACA,WAAehB"}

package/dist/esm/cli/find.d.ts

@@ -1,3 +0,0 @@
-import type { Command } from './types.js';
-declare const findCmd: Command;
-export default findCmd;

package/dist/esm/cli/find.js

@@ -1,14 +0,0 @@
-import { find } from '../commands.js';
-import { printRows } from '../output.js';
-import { parseK, withDb } from './shared.js';
-const findCmd = async (ctx)=>{
-    const [terms] = ctx.rest;
-    if (!terms) ctx.usageError(`usage: ${ctx.name} find "<terms>" [--where "<sql>"] [--k n] [--semantic]`);
-    const k = parseK(ctx);
-    await withDb(ctx, async (db, cfg)=>printRows(await find(db, cfg, terms, {
-            k,
-            where: ctx.values.where,
-            semantic: ctx.values.semantic
-        }), ctx.format));
-};
-export default findCmd;

package/dist/esm/cli/find.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/cli/find.ts"],"sourcesContent":["import { find } from '../commands.ts';\nimport { printRows } from '../output.ts';\nimport { parseK, withDb } from './shared.ts';\nimport type { Command } from './types.ts';\n\nconst findCmd: Command = async (ctx) => {\n  const [terms] = ctx.rest;\n  if (!terms) ctx.usageError(`usage: ${ctx.name} find \"<terms>\" [--where \"<sql>\"] [--k n] [--semantic]`);\n  const k = parseK(ctx);\n  await withDb(ctx, async (db, cfg) => printRows(await find(db, cfg, terms, { k, where: ctx.values.where, semantic: ctx.values.semantic }), ctx.format));\n};\nexport default findCmd;\n"],"names":["find","printRows","parseK","withDb","findCmd","ctx","terms","rest","usageError","name","k","db","cfg","where","values","semantic","format"],"mappings":"AAAA,SAASA,IAAI,QAAQ,iBAAiB;AACtC,SAASC,SAAS,QAAQ,eAAe;AACzC,SAASC,MAAM,EAAEC,MAAM,QAAQ,cAAc;AAG7C,MAAMC,UAAmB,OAAOC;IAC9B,MAAM,CAACC,MAAM,GAAGD,IAAIE,IAAI;IACxB,IAAI,CAACD,OAAOD,IAAIG,UAAU,CAAC,CAAC,OAAO,EAAEH,IAAII,IAAI,CAAC,sDAAsD,CAAC;IACrG,MAAMC,IAAIR,OAAOG;IACjB,MAAMF,OAAOE,KAAK,OAAOM,IAAIC,MAAQX,UAAU,MAAMD,KAAKW,IAAIC,KAAKN,OAAO;YAAEI;YAAGG,OAAOR,IAAIS,MAAM,CAACD,KAAK;YAAEE,UAAUV,IAAIS,MAAM,CAACC,QAAQ;QAAC,IAAIV,IAAIW,MAAM;AACtJ;AACA,eAAeZ,QAAQ"}