projview 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -2
- package/bin/projview.js +46 -20
- package/docs/README.md +1 -0
- package/docs/ai-tests/search.md +120 -0
- package/docs/how-to-set-up.md +51 -2
- package/docs/reference/api.md +36 -2
- package/package.json +5 -2
- package/src/search.js +30 -7
- package/src/server.js +11 -4
- package/src/store.js +189 -0
package/README.md
CHANGED
|
@@ -26,15 +26,55 @@ projview ./docs --port 5000
|
|
|
26
26
|
projview /path/to/notes --no-open
|
|
27
27
|
```
|
|
28
28
|
|
|
29
|
+
## use it from scripts or an AI
|
|
30
|
+
|
|
31
|
+
projview also has a **machine interface** - subcommands that print JSON to stdout
|
|
32
|
+
and exit (no server, no browser), so an agent or script can query your docs:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
projview search "<query>" [path] # ranked: filename matches + in-content hits w/ line snippets
|
|
36
|
+
projview tree [path] # the file tree as JSON
|
|
37
|
+
projview search "mermaid" --demo # quick test on the bundled docs
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
```jsonc
|
|
41
|
+
// projview search "live reload"
|
|
42
|
+
{ "query": "live reload",
|
|
43
|
+
"files": [ /* fuzzy filename matches: {path, name, score} */ ],
|
|
44
|
+
"content": [ { "path": "reference/api.md", "count": 4,
|
|
45
|
+
"snippets": [ { "line": 68, "text": "### live reload" } ] } ] }
|
|
46
|
+
```
|
|
47
|
+
|
|
29
48
|
## what it does
|
|
30
49
|
|
|
31
50
|
| step | detail |
|
|
32
51
|
| --- | --- |
|
|
33
52
|
| walk | recursively finds `.md` / `.mmd`, skips `node_modules`, `dist`, dot-dirs |
|
|
34
|
-
| index | builds an in-memory tree (nothing written to disk) |
|
|
53
|
+
| index | builds an in-memory tree (nothing written to disk by default) |
|
|
35
54
|
| serve | spawns a localhost site + opens your browser |
|
|
36
55
|
| live | edits to files reload in the browser automatically |
|
|
37
56
|
|
|
38
|
-
> ctrl-c and it's gone
|
|
57
|
+
> ephemeral by default - ctrl-c and it's gone. opt into persistence below.
|
|
58
|
+
|
|
59
|
+
## persistence (opt-in)
|
|
60
|
+
|
|
61
|
+
By default the index is in-memory and nothing is written to disk. Opt in for a
|
|
62
|
+
faster warm start, or a queryable store an AI can read:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
projview docs --persist # sqlite cache in ~/.projview/cache
|
|
66
|
+
projview docs --store-url "sqlite:///tmp/docs.sqlite" # a sqlite file you choose
|
|
67
|
+
projview docs --store-url "$DATABASE_URL" # your own postgres (needs `pg`)
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
| backend | how |
|
|
71
|
+
| --- | --- |
|
|
72
|
+
| `memory` (default) | _(nothing)_ - ephemeral |
|
|
73
|
+
| `lite` (sqlite) | `--persist` |
|
|
74
|
+
| `superlite` (JSON) | `--store superlite --persist` |
|
|
75
|
+
| `custom` (sqlite/postgres URL) | `--store-url <url>` (or `DATABASE_URL`) |
|
|
76
|
+
|
|
77
|
+
On re-run it reloads the cache and only re-reads files whose mtime changed.
|
|
78
|
+
Connection strings come from env/`~/.projview` - never commit credentials.
|
|
39
79
|
|
|
40
80
|
see [samples/architecture.md](samples/architecture.md) for a mermaid demo.
|
package/bin/projview.js
CHANGED
|
@@ -6,6 +6,7 @@ import { readFileSync } from 'node:fs';
|
|
|
6
6
|
import { startServer } from '../src/server.js';
|
|
7
7
|
import { buildTree, countFiles } from '../src/walk.js';
|
|
8
8
|
import { buildIndex, search } from '../src/search.js';
|
|
9
|
+
import { resolveStoreConfig, createStore } from '../src/store.js';
|
|
9
10
|
|
|
10
11
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
11
12
|
const DEMO_DIR = path.join(__dirname, '..', 'docs'); // bundled sample docs
|
|
@@ -24,6 +25,9 @@ function parseArgs(argv) {
|
|
|
24
25
|
if (arg === '--port' || arg === '-p') opts.port = Number(argv[++i]) || opts.port;
|
|
25
26
|
else if (arg === '--no-open') opts.open = false;
|
|
26
27
|
else if (arg === '--demo') opts.demo = true;
|
|
28
|
+
else if (arg === '--store') opts.store = argv[++i];
|
|
29
|
+
else if (arg === '--store-url') opts.storeUrl = argv[++i];
|
|
30
|
+
else if (arg === '--persist') opts.persist = true;
|
|
27
31
|
else if (arg === '--version' || arg === '-v') opts.version = true;
|
|
28
32
|
else if (arg === '--help' || arg === '-h') opts.help = true;
|
|
29
33
|
else if (arg.startsWith('-')) opts.unknown = arg; // unrecognised flag -> error, don't treat as a path
|
|
@@ -36,27 +40,42 @@ function parseArgs(argv) {
|
|
|
36
40
|
|
|
37
41
|
// --- machine interface: subcommands that print JSON to stdout and exit ---
|
|
38
42
|
|
|
39
|
-
//
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
+
// parse subcommand args: separate positionals from flags (consumes flag values,
|
|
44
|
+
// so e.g. a postgres:// store URL is never mistaken for a path)
|
|
45
|
+
function parseSubArgs(args) {
|
|
46
|
+
const o = { positional: [] };
|
|
47
|
+
for (let i = 0; i < args.length; i++) {
|
|
48
|
+
const a = args[i];
|
|
49
|
+
if (a === '--store') o.store = args[++i];
|
|
50
|
+
else if (a === '--store-url') o.storeUrl = args[++i];
|
|
51
|
+
else if (a === '--persist') o.persist = true;
|
|
52
|
+
else if (a === '--demo') o.demo = true;
|
|
53
|
+
else if (a.startsWith('-')) { /* ignore unknown flag in a subcommand */ }
|
|
54
|
+
else o.positional.push(a);
|
|
55
|
+
}
|
|
56
|
+
return o;
|
|
43
57
|
}
|
|
44
58
|
|
|
59
|
+
const subRoot = (o, idx) =>
|
|
60
|
+
o.demo ? DEMO_DIR : (o.positional[idx] ? path.resolve(process.cwd(), o.positional[idx]) : process.cwd());
|
|
61
|
+
|
|
45
62
|
async function cmdSearch(args) {
|
|
46
|
-
const
|
|
47
|
-
const query = positional[0];
|
|
63
|
+
const o = parseSubArgs(args);
|
|
64
|
+
const query = o.positional[0];
|
|
48
65
|
if (!query) {
|
|
49
|
-
console.error('usage: projview search <query> [path] [--demo]');
|
|
66
|
+
console.error('usage: projview search <query> [path] [--demo] [--store <kind>|--store-url <url>] [--persist]');
|
|
50
67
|
process.exit(1);
|
|
51
68
|
}
|
|
52
|
-
const root =
|
|
53
|
-
|
|
69
|
+
const root = subRoot(o, 1);
|
|
70
|
+
const store = createStore(resolveStoreConfig(o), root);
|
|
71
|
+
await buildIndex(root, store);
|
|
54
72
|
process.stdout.write(JSON.stringify(search(query), null, 2) + '\n');
|
|
73
|
+
await store.close();
|
|
55
74
|
}
|
|
56
75
|
|
|
57
76
|
function cmdTree(args) {
|
|
58
|
-
const
|
|
59
|
-
const root =
|
|
77
|
+
const o = parseSubArgs(args);
|
|
78
|
+
const root = subRoot(o, 0);
|
|
60
79
|
const tree = buildTree(root);
|
|
61
80
|
process.stdout.write(JSON.stringify({ root: path.basename(root) || root, files: countFiles(tree), tree }, null, 2) + '\n');
|
|
62
81
|
}
|
|
@@ -84,13 +103,17 @@ arguments:
|
|
|
84
103
|
path directory to index (default: current directory)
|
|
85
104
|
|
|
86
105
|
options:
|
|
87
|
-
-p, --port <n>
|
|
88
|
-
--demo
|
|
89
|
-
--no-open
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
106
|
+
-p, --port <n> preferred port (default: 4321, climbs if taken)
|
|
107
|
+
--demo preview projview's bundled sample docs
|
|
108
|
+
--no-open do not auto-open the browser
|
|
109
|
+
--persist keep the index in ~/.projview/cache and reuse it next run
|
|
110
|
+
--store <kind> store backend: memory | superlite | lite (default: memory)
|
|
111
|
+
--store-url <url> custom store: sqlite://<file> or postgres://<...> (or set DATABASE_URL)
|
|
112
|
+
-v, --version print the version and exit
|
|
113
|
+
-h, --help show this help
|
|
114
|
+
|
|
115
|
+
ephemeral by default - nothing is persisted unless you opt in with --persist /
|
|
116
|
+
--store / --store-url. ctrl-c removes any ephemeral store.`;
|
|
94
117
|
|
|
95
118
|
async function main() {
|
|
96
119
|
const argv = process.argv.slice(2);
|
|
@@ -107,12 +130,15 @@ async function main() {
|
|
|
107
130
|
process.exit(1);
|
|
108
131
|
}
|
|
109
132
|
|
|
110
|
-
const
|
|
133
|
+
const storeConfig = resolveStoreConfig(opts);
|
|
134
|
+
const { port, close, storeInfo } = await startServer(opts.root, { port: opts.port, host: opts.host, store: storeConfig });
|
|
111
135
|
const url = `http://${opts.host}:${port}`;
|
|
112
136
|
|
|
113
137
|
console.log(`\n projview -> ${url}`);
|
|
114
138
|
console.log(` indexing -> ${opts.root}`);
|
|
115
|
-
console.log(`
|
|
139
|
+
console.log(` store -> ${storeInfo.describe}`);
|
|
140
|
+
if (storeInfo.warm) console.log(` warm -> reused cache, reindexed ${storeInfo.reindexed}/${storeInfo.total} changed`);
|
|
141
|
+
console.log(` (ctrl-c to stop)\n`);
|
|
116
142
|
|
|
117
143
|
if (opts.open) openBrowser(url);
|
|
118
144
|
|
package/docs/README.md
CHANGED
|
@@ -21,6 +21,7 @@ faults. use this tree while iterating on the tool.
|
|
|
21
21
|
| edge | [syntax stress](edge-cases/syntax-stress.md) | gnarly markdown + inline broken fence |
|
|
22
22
|
| edge | [faulty refs](edge-cases/faulty-refs.md) | **several dead links** on purpose |
|
|
23
23
|
| edge | [buried file](edge-cases/deep/nested/buried.md) | deep nesting - indentation + auto-expand |
|
|
24
|
+
| ai-tests | [search test](ai-tests/search.md) | AI-run test of the `projview search` machine interface |
|
|
24
25
|
|
|
25
26
|
## intentional faults (should NOT crash the app)
|
|
26
27
|
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# search functionality - AI test
|
|
2
|
+
|
|
3
|
+
An AI-driven test of projview's `search` **machine interface** (`projview search
|
|
4
|
+
<query>` → JSON on stdout). The goal: probe distinct behaviours of the engine -
|
|
5
|
+
content search, fuzzy filenames, multi-term logic, ranking, and edge cases - by
|
|
6
|
+
running the published CLI and inspecting the JSON it returns.
|
|
7
|
+
|
|
8
|
+
## experimental parameters
|
|
9
|
+
|
|
10
|
+
| parameter | value |
|
|
11
|
+
| --- | --- |
|
|
12
|
+
| tool | `projview` (global install) |
|
|
13
|
+
| version | `0.4.0` |
|
|
14
|
+
| command form | `projview search "<query>" --demo` |
|
|
15
|
+
| dataset | the bundled demo docs (`--demo`) - **14 files** at time of test |
|
|
16
|
+
| output | JSON: `{ query, files[], content[] }` parsed from stdout |
|
|
17
|
+
| ranking | filenames by fuzzy subsequence score; contents by term frequency |
|
|
18
|
+
|
|
19
|
+
> Note: this very report is the 15th file in the set, so re-running the probes
|
|
20
|
+
> below will now also surface `ai-tests/search.md` itself.
|
|
21
|
+
|
|
22
|
+
## method
|
|
23
|
+
|
|
24
|
+
Five search terms were chosen, each targeting a different aspect of the engine:
|
|
25
|
+
|
|
26
|
+
| # | term | aspect under test |
|
|
27
|
+
| --- | --- | --- |
|
|
28
|
+
| 1 | `live reload` | content search across multiple files + snippet line numbers |
|
|
29
|
+
| 2 | `mermaid` | frequency ranking + case-insensitivity (`MERMAID` vs `mermaid`) |
|
|
30
|
+
| 3 | `cfg` | fuzzy **filename** match (subsequence), independent of content |
|
|
31
|
+
| 4 | `broken diagram` | multi-term **AND** logic + cross-file ranking |
|
|
32
|
+
| 5 | `kubernetes` | graceful **no-match** behaviour |
|
|
33
|
+
|
|
34
|
+
## results
|
|
35
|
+
|
|
36
|
+
### 1. content search - `live reload`
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
projview search "live reload" --demo
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Three files matched, ranked by frequency, each snippet carrying a line number:
|
|
43
|
+
|
|
44
|
+
| file | hits | top snippet |
|
|
45
|
+
| --- | --- | --- |
|
|
46
|
+
| `reference/api.md` | 4 | L68 `### live reload` |
|
|
47
|
+
| `diagrams/sequence.mmd` | 2 | L13 `edit a file -> SSE -> live reload` |
|
|
48
|
+
| `reference/glossary.md` | 2 | L9 `live reload - SSE-driven refresh` |
|
|
49
|
+
|
|
50
|
+
✅ Multi-file content search, frequency-ordered, with usable `file:line` citations.
|
|
51
|
+
|
|
52
|
+
### 2. ranking + case-insensitivity - `mermaid`
|
|
53
|
+
|
|
54
|
+
`mermaid` returned **8** content matches. Crucially, `MERMAID` (uppercase)
|
|
55
|
+
returned the **same 8** - the query is lower-cased before matching.
|
|
56
|
+
|
|
57
|
+
✅ Case-insensitive; ranked by occurrence count.
|
|
58
|
+
|
|
59
|
+
### 3. fuzzy filename match - `cfg`
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
projview search "cfg" --demo
|
|
63
|
+
# files: ["guides/configuration.md"]
|
|
64
|
+
# content: (none)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
`cfg` is not a substring of any document body, but it **is** a subsequence of
|
|
68
|
+
`configuration.md`, so it surfaces in the `files` block while `content` stays empty.
|
|
69
|
+
|
|
70
|
+
✅ Filename matching is fuzzy (subsequence) and independent of content search.
|
|
71
|
+
|
|
72
|
+
### 4. multi-term AND + ranking - `broken diagram`
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
projview search "broken diagram" --demo
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
| file | hits |
|
|
79
|
+
| --- | --- |
|
|
80
|
+
| `README.md` | 12 |
|
|
81
|
+
| `guides/troubleshooting.md` | 7 |
|
|
82
|
+
| `edge-cases/syntax-stress.md` | 6 |
|
|
83
|
+
| `guides/configuration.md` | 3 |
|
|
84
|
+
| `edge-cases/faulty-refs.md` | 2 |
|
|
85
|
+
|
|
86
|
+
Only files containing **both** terms appear; ranking is by total term frequency.
|
|
87
|
+
|
|
88
|
+
✅ Multi-term is a **file-level AND** (every term must appear somewhere in the
|
|
89
|
+
file), ranked by combined frequency.
|
|
90
|
+
|
|
91
|
+
### 5. graceful no-match - `kubernetes`
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
projview search "kubernetes" --demo
|
|
95
|
+
# { "query": "kubernetes", "files": [], "content": [] }
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
✅ Empty arrays, exit code `0` - a clean "nothing found", not an error.
|
|
99
|
+
|
|
100
|
+
## observations & limitations
|
|
101
|
+
|
|
102
|
+
- **Case-insensitive** matching (query is lower-cased).
|
|
103
|
+
- **Multi-term = file-level AND**, not phrase match - the words can be anywhere in
|
|
104
|
+
the file, ranked by total frequency.
|
|
105
|
+
- **Snippets are first-occurrence**: the shown snippet is the first line containing
|
|
106
|
+
*any* query term, which isn't always the most relevant line for multi-term
|
|
107
|
+
queries. (A future improvement: prefer lines matching the most terms.)
|
|
108
|
+
- **Two independent result blocks**: `files` (fuzzy filename) and `content`
|
|
109
|
+
(full-text) - either can be populated without the other.
|
|
110
|
+
|
|
111
|
+
## reproduce
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
npm i -g projview@latest
|
|
115
|
+
projview search "live reload" --demo
|
|
116
|
+
projview search "cfg" --demo
|
|
117
|
+
projview search "broken diagram" --demo
|
|
118
|
+
projview search "kubernetes" --demo
|
|
119
|
+
projview tree --demo # list the dataset
|
|
120
|
+
```
|
package/docs/how-to-set-up.md
CHANGED
|
@@ -66,7 +66,7 @@ npm install
|
|
|
66
66
|
npm run demo # preview the bundled docs/ (alias for --demo)
|
|
67
67
|
npm start # index the current directory
|
|
68
68
|
node bin/projview.js ./docs # or call the CLI directly
|
|
69
|
-
npm test # run the test suite (node:test,
|
|
69
|
+
npm test # run the test suite (node:test, 29 tests)
|
|
70
70
|
```
|
|
71
71
|
|
|
72
72
|
See the [project README](README.md) for the architecture and the
|
|
@@ -74,12 +74,30 @@ See the [project README](README.md) for the architecture and the
|
|
|
74
74
|
|
|
75
75
|
## CLI reference
|
|
76
76
|
|
|
77
|
+
**Commands**
|
|
78
|
+
|
|
79
|
+
| command | meaning |
|
|
80
|
+
| --- | --- |
|
|
81
|
+
| `projview [path]` | start the viewer (default) |
|
|
82
|
+
| `projview search <query> [path]` | print ranked search results as JSON, then exit |
|
|
83
|
+
| `projview tree [path]` | print the file tree as JSON, then exit |
|
|
84
|
+
|
|
85
|
+
The `search` and `tree` subcommands are projview's **machine interface** - JSON on
|
|
86
|
+
stdout, no server, no browser - built for agents and scripts. See
|
|
87
|
+
[ai-tests/search.md](ai-tests/search.md) for a worked test.
|
|
88
|
+
|
|
89
|
+
**Options**
|
|
90
|
+
|
|
77
91
|
| argument / flag | meaning |
|
|
78
92
|
| --- | --- |
|
|
79
93
|
| `[path]` | directory to index (default: current directory) |
|
|
80
|
-
| `--demo` |
|
|
94
|
+
| `--demo` | use projview's own bundled sample docs (works with the subcommands too) |
|
|
81
95
|
| `-p`, `--port <n>` | preferred port (default `4321`, climbs if taken) |
|
|
82
96
|
| `--no-open` | don't auto-open the browser |
|
|
97
|
+
| `--persist` | keep the index in `~/.projview/cache` and reuse it next run |
|
|
98
|
+
| `--store <kind>` | store backend: `memory` \| `superlite` \| `lite` (default: `memory`; `lite` when `--persist`) |
|
|
99
|
+
| `--store-url <url>` | use a custom store: `sqlite://<file>` or `postgres://<...>` |
|
|
100
|
+
| `-v`, `--version` | print the version and exit |
|
|
83
101
|
| `-h`, `--help` | show help |
|
|
84
102
|
|
|
85
103
|
**Environment variables** (handy for containers / servers):
|
|
@@ -88,6 +106,37 @@ See the [project README](README.md) for the architecture and the
|
|
|
88
106
|
| --- | --- | --- |
|
|
89
107
|
| `PORT` | `4321` | port to serve on |
|
|
90
108
|
| `HOST` | `127.0.0.1` | interface to bind (`0.0.0.0` to expose) |
|
|
109
|
+
| `PROJVIEW_PERSIST` | `0` | `1` to persist the index (same as `--persist`) |
|
|
110
|
+
| `PROJVIEW_STORE` | `memory` | store backend (same as `--store`) |
|
|
111
|
+
| `PROJVIEW_STORE_URL` / `DATABASE_URL` | — | custom store connection URL (same as `--store-url`) |
|
|
112
|
+
|
|
113
|
+
## persistence & stores
|
|
114
|
+
|
|
115
|
+
By default projview is **ephemeral** - the index lives in memory and nothing is
|
|
116
|
+
written to disk. Opt in to persistence when you want a faster warm start or a
|
|
117
|
+
queryable store an AI can read:
|
|
118
|
+
|
|
119
|
+
| backend | what | how |
|
|
120
|
+
| --- | --- | --- |
|
|
121
|
+
| `memory` | in-memory, nothing persisted (**default**) | _(no flag)_ |
|
|
122
|
+
| `lite` | sqlite via Node's built-in `node:sqlite` | `--persist` (or `--store lite`) |
|
|
123
|
+
| `superlite` | a plain JSON file | `--store superlite --persist` |
|
|
124
|
+
| `custom` | your own DB via a connection URL | `--store-url sqlite://./my.sqlite` or `postgres://…` |
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
projview docs --persist # warm-start cache in ~/.projview/cache
|
|
128
|
+
projview docs --store-url "sqlite:///tmp/docs.sqlite" # a sqlite file you choose
|
|
129
|
+
projview docs --store-url "$DATABASE_URL" # your postgres (needs the `pg` package)
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
- **Locations:** persisted stores live in `~/.projview/cache/`; ephemeral ones in
|
|
133
|
+
`~/.projview/ephemeral/` and are removed on ctrl-c. Defaults can be set in
|
|
134
|
+
`~/.projview/config.json`.
|
|
135
|
+
- **Warm start:** on re-run, projview loads the cached index and only re-reads
|
|
136
|
+
files whose modification time changed.
|
|
137
|
+
- **Postgres** needs the optional `pg` package (`npm i pg`). The connection URL
|
|
138
|
+
comes from `--store-url` / `DATABASE_URL` / your config file - **never commit a
|
|
139
|
+
connection string with credentials**; keep it in the environment.
|
|
91
140
|
|
|
92
141
|
## use the source directly (advanced)
|
|
93
142
|
|
package/docs/reference/api.md
CHANGED
|
@@ -21,6 +21,22 @@ Renders a single file to HTML on demand. Query param `p` is the relative path.
|
|
|
21
21
|
|
|
22
22
|
Server-sent events stream for live reload.
|
|
23
23
|
|
|
24
|
+
## machine interface (CLI)
|
|
25
|
+
|
|
26
|
+
Besides the HTTP server, two subcommands print JSON to stdout and exit - no
|
|
27
|
+
server, no browser - for agents and scripts.
|
|
28
|
+
|
|
29
|
+
### projview search
|
|
30
|
+
|
|
31
|
+
`projview search "<query>" [path]` returns `{ query, files[], content[] }`.
|
|
32
|
+
Filenames match fuzzily (subsequence); contents are ranked by term frequency with
|
|
33
|
+
line-numbered snippets. See [the AI search test](../ai-tests/search.md).
|
|
34
|
+
|
|
35
|
+
### projview tree
|
|
36
|
+
|
|
37
|
+
`projview tree [path]` returns `{ root, files, tree[] }` - the same structure
|
|
38
|
+
`/api/tree` serves.
|
|
39
|
+
|
|
24
40
|
## the walker
|
|
25
41
|
|
|
26
42
|
### ignored directories
|
|
@@ -59,7 +75,7 @@ Requested paths are resolved and confined to the root.
|
|
|
59
75
|
|
|
60
76
|
### tree nav
|
|
61
77
|
|
|
62
|
-
Collapsible dirs,
|
|
78
|
+
Collapsible dirs, cmd+K search palette, auto-expand on navigate.
|
|
63
79
|
|
|
64
80
|
### viewer
|
|
65
81
|
|
|
@@ -69,9 +85,27 @@ On-demand fetch + render, in-app link routing, anchor jumps.
|
|
|
69
85
|
|
|
70
86
|
A chokidar watcher broadcasts change/tree events over SSE.
|
|
71
87
|
|
|
88
|
+
## stores (persistence)
|
|
89
|
+
|
|
90
|
+
The index is in-memory by default (ephemeral). Opt into a pluggable store with
|
|
91
|
+
`--persist` / `--store` / `--store-url`:
|
|
92
|
+
|
|
93
|
+
### backends
|
|
94
|
+
|
|
95
|
+
`memory` (default) · `superlite` (JSON) · `lite` (sqlite via `node:sqlite`) ·
|
|
96
|
+
`custom` (a `sqlite://` or `postgres://` connection URL; postgres needs the
|
|
97
|
+
optional `pg` package).
|
|
98
|
+
|
|
99
|
+
### warm start
|
|
100
|
+
|
|
101
|
+
A persisted store lives in `~/.projview/cache`; on re-run projview reloads it and
|
|
102
|
+
re-reads only files whose mtime changed. Connection strings come from the
|
|
103
|
+
environment / config - never committed.
|
|
104
|
+
|
|
72
105
|
## performance notes
|
|
73
106
|
|
|
74
|
-
Indexing is
|
|
107
|
+
Indexing is a directory walk; persistence is opt-in (see stores). Search runs
|
|
108
|
+
in-memory regardless of backend.
|
|
75
109
|
|
|
76
110
|
## limitations
|
|
77
111
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "projview",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.1",
|
|
4
4
|
"description": "Ephemeral local previewer for markdown + mermaid files in any directory",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
"docs"
|
|
19
19
|
],
|
|
20
20
|
"engines": {
|
|
21
|
-
"node": ">=
|
|
21
|
+
"node": ">=22"
|
|
22
22
|
},
|
|
23
23
|
"keywords": [
|
|
24
24
|
"markdown",
|
|
@@ -42,6 +42,9 @@
|
|
|
42
42
|
"highlight.js": "^11.11.1",
|
|
43
43
|
"markdown-it": "^14.1.0"
|
|
44
44
|
},
|
|
45
|
+
"optionalDependencies": {
|
|
46
|
+
"pg": "^8.21.0"
|
|
47
|
+
},
|
|
45
48
|
"devDependencies": {
|
|
46
49
|
"svg-pan-zoom": "^3.6.2"
|
|
47
50
|
}
|
package/src/search.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import fsp from 'node:fs/promises';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { buildTree } from './walk.js';
|
|
4
|
+
import { createStore } from './store.js';
|
|
4
5
|
|
|
5
|
-
// In-memory index: relPath -> { path, name, content, lower }.
|
|
6
|
-
//
|
|
6
|
+
// In-memory index: relPath -> { path, name, content, lower }. Search always runs
|
|
7
|
+
// over this; a Store (optional, opt-in) just persists/warm-starts the entries.
|
|
7
8
|
const index = new Map();
|
|
8
9
|
let theRoot = null;
|
|
9
10
|
|
|
@@ -19,15 +20,37 @@ function addToIndex(rel, content) {
|
|
|
19
20
|
index.set(rel, { path: rel, name: rel.split('/').pop(), content, lower: content.toLowerCase() });
|
|
20
21
|
}
|
|
21
22
|
|
|
22
|
-
//
|
|
23
|
-
|
|
23
|
+
// Build the index for `root`. With a persisted store, loads the cached entries
|
|
24
|
+
// and only re-reads files whose mtime changed (warm start); else reads them all.
|
|
25
|
+
// Returns the store + a small summary, so the caller can close it on shutdown.
|
|
26
|
+
export async function buildIndex(root, store) {
|
|
24
27
|
theRoot = root;
|
|
25
28
|
index.clear();
|
|
29
|
+
store = store || createStore({ kind: 'memory' }, root);
|
|
30
|
+
|
|
31
|
+
const cached = await store.load();
|
|
32
|
+
const cachedByPath = new Map((cached?.entries || []).map((e) => [e.path, e]));
|
|
26
33
|
const files = flattenFiles(buildTree(root));
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
34
|
+
let reindexed = 0;
|
|
35
|
+
|
|
36
|
+
const entries = await Promise.all(files.map(async (f) => {
|
|
37
|
+
const abs = path.join(root, f.path);
|
|
38
|
+
let mtime = 0;
|
|
39
|
+
try { mtime = (await fsp.stat(abs)).mtimeMs; } catch { /* gone */ }
|
|
40
|
+
const prev = cachedByPath.get(f.path);
|
|
41
|
+
let content;
|
|
42
|
+
if (prev && prev.mtime === mtime) {
|
|
43
|
+
content = prev.content; // unchanged -> reuse cached
|
|
44
|
+
} else {
|
|
45
|
+
reindexed++;
|
|
46
|
+
try { content = await fsp.readFile(abs, 'utf8'); } catch { content = ''; }
|
|
47
|
+
}
|
|
48
|
+
return { path: f.path, name: f.name, ext: f.ext, mtime, content };
|
|
30
49
|
}));
|
|
50
|
+
|
|
51
|
+
for (const e of entries) addToIndex(e.path, e.content);
|
|
52
|
+
await store.save(entries);
|
|
53
|
+
return { store, total: entries.length, reindexed, warm: !!cached };
|
|
31
54
|
}
|
|
32
55
|
|
|
33
56
|
export async function updateFile(rel) {
|
package/src/server.js
CHANGED
|
@@ -8,6 +8,7 @@ import chokidar from 'chokidar';
|
|
|
8
8
|
import { buildTree, isIgnoredDir, isPreviewable } from './walk.js';
|
|
9
9
|
import { renderFile } from './render.js';
|
|
10
10
|
import { buildIndex, updateFile, removeFile, search } from './search.js';
|
|
11
|
+
import { createStore } from './store.js';
|
|
11
12
|
|
|
12
13
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
13
14
|
const PUBLIC_DIR = path.join(__dirname, '..', 'public');
|
|
@@ -46,7 +47,7 @@ function resolveInRoot(root, relPath) {
|
|
|
46
47
|
return abs;
|
|
47
48
|
}
|
|
48
49
|
|
|
49
|
-
export async function startServer(root, { port = 4321, host = '127.0.0.1' } = {}) {
|
|
50
|
+
export async function startServer(root, { port = 4321, host = '127.0.0.1', store: storeConfig } = {}) {
|
|
50
51
|
const sseClients = new Set();
|
|
51
52
|
|
|
52
53
|
function broadcast(event, payload) {
|
|
@@ -128,16 +129,22 @@ export async function startServer(root, { port = 4321, host = '127.0.0.1' } = {}
|
|
|
128
129
|
.on('addDir', () => broadcast('tree'))
|
|
129
130
|
.on('unlinkDir', () => broadcast('tree'));
|
|
130
131
|
|
|
131
|
-
|
|
132
|
+
const store = createStore(storeConfig, root);
|
|
133
|
+
const indexInfo = await buildIndex(root, store); // load-or-build the search index
|
|
132
134
|
const listenPort = await listenWithRetry(server, port, host);
|
|
133
135
|
|
|
134
136
|
const close = () => new Promise((resolve) => {
|
|
135
137
|
watcher.close();
|
|
136
138
|
for (const res of sseClients) res.end();
|
|
137
|
-
server.close(() => resolve());
|
|
139
|
+
Promise.resolve(store.close()).finally(() => server.close(() => resolve()));
|
|
138
140
|
});
|
|
139
141
|
|
|
140
|
-
return {
|
|
142
|
+
return {
|
|
143
|
+
server,
|
|
144
|
+
port: listenPort,
|
|
145
|
+
close,
|
|
146
|
+
storeInfo: { describe: store.describe(), warm: indexInfo.warm, reindexed: indexInfo.reindexed, total: indexInfo.total }
|
|
147
|
+
};
|
|
141
148
|
}
|
|
142
149
|
|
|
143
150
|
// Try `port`, then climb until one is free (give up after a sane range).
|
package/src/store.js
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/* pluggable persistence for the index.
|
|
2
|
+
*
|
|
3
|
+
* A Store persists indexed entries so a re-spawn can warm-start. Backends:
|
|
4
|
+
* memory - no persistence (ephemeral default; nothing written)
|
|
5
|
+
* superlite - a JSON file
|
|
6
|
+
* lite - sqlite via node:sqlite (built-in)
|
|
7
|
+
* custom - a connection URL: sqlite://<file> or postgres://<...> (lazy `pg`)
|
|
8
|
+
*
|
|
9
|
+
* An entry is { path, name, ext, mtime, content }. Everything above the store
|
|
10
|
+
* stays storage-agnostic - search still runs in memory over loaded entries.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import fs from 'node:fs';
|
|
14
|
+
import fsp from 'node:fs/promises';
|
|
15
|
+
import path from 'node:path';
|
|
16
|
+
import os from 'node:os';
|
|
17
|
+
import { createHash } from 'node:crypto';
|
|
18
|
+
|
|
19
|
+
// node:sqlite is flagged "experimental" and emits a noisy warning on use; we
|
|
20
|
+
// opted into it deliberately, so quiet just that one notice (others pass through).
|
|
21
|
+
const _emitWarning = process.emitWarning.bind(process);
|
|
22
|
+
process.emitWarning = (warning, ...rest) => {
|
|
23
|
+
if (typeof warning === 'string' && warning.includes('SQLite is an experimental feature')) return;
|
|
24
|
+
return _emitWarning(warning, ...rest);
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const HOME = path.join(os.homedir(), '.projview');
|
|
28
|
+
const CACHE_DIR = path.join(HOME, 'cache'); // retained (--persist)
|
|
29
|
+
const EPHEMERAL_DIR = path.join(HOME, 'ephemeral'); // cleaned up on close
|
|
30
|
+
const CONFIG_FILE = path.join(HOME, 'config.json'); // default flags
|
|
31
|
+
|
|
32
|
+
const keyFor = (root) => createHash('sha256').update(path.resolve(root)).digest('hex').slice(0, 16);
|
|
33
|
+
const ensureDir = (d) => fs.mkdirSync(d, { recursive: true });
|
|
34
|
+
const redact = (url) => url.replace(/\/\/[^@/]*@/, '//***@'); // hide creds in logs
|
|
35
|
+
|
|
36
|
+
function readConfigFile() {
|
|
37
|
+
try { return JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8')); } catch { return {}; }
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Resolve store config: CLI flags > env > ~/.projview/config.json > defaults.
|
|
41
|
+
export function resolveStoreConfig(flags = {}) {
|
|
42
|
+
const env = process.env;
|
|
43
|
+
const file = readConfigFile();
|
|
44
|
+
const pick = (...vals) => vals.find((v) => v !== undefined);
|
|
45
|
+
|
|
46
|
+
const url = pick(flags.storeUrl, env.PROJVIEW_STORE_URL, env.DATABASE_URL, file.storeUrl);
|
|
47
|
+
const persist = pick(
|
|
48
|
+
flags.persist,
|
|
49
|
+
['1', 'true'].includes(env.PROJVIEW_PERSIST) ? true : undefined,
|
|
50
|
+
file.persist
|
|
51
|
+
) || false;
|
|
52
|
+
let kind = pick(flags.store, env.PROJVIEW_STORE, file.store);
|
|
53
|
+
if (!kind) kind = url ? 'custom' : persist ? 'lite' : 'memory';
|
|
54
|
+
return { kind, url, persist };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function createStore(config, root) {
|
|
58
|
+
const cfg = config || { kind: 'memory' };
|
|
59
|
+
switch (cfg.kind) {
|
|
60
|
+
case 'memory': return memoryStore();
|
|
61
|
+
case 'superlite': return superliteStore(root, cfg.persist);
|
|
62
|
+
case 'lite': return sqliteAt(fileFor(root, cfg.persist, 'sqlite'), { cleanup: !cfg.persist });
|
|
63
|
+
case 'custom': return customStore(cfg.url);
|
|
64
|
+
default: throw new Error(`unknown store kind: ${cfg.kind}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function fileFor(root, persist, ext) {
|
|
69
|
+
const dir = persist ? CACHE_DIR : EPHEMERAL_DIR;
|
|
70
|
+
ensureDir(dir);
|
|
71
|
+
return path.join(dir, `${keyFor(root)}.${ext}`);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/* ---------- backends ---------- */
|
|
75
|
+
|
|
76
|
+
function memoryStore() {
|
|
77
|
+
return {
|
|
78
|
+
kind: 'memory',
|
|
79
|
+
async load() { return null; },
|
|
80
|
+
async save() {},
|
|
81
|
+
async close() {},
|
|
82
|
+
describe() { return 'in-memory (ephemeral - nothing persisted)'; }
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function superliteStore(root, persist) {
|
|
87
|
+
const file = fileFor(root, persist, 'json');
|
|
88
|
+
return {
|
|
89
|
+
kind: 'superlite',
|
|
90
|
+
file,
|
|
91
|
+
async load() {
|
|
92
|
+
try { return JSON.parse(await fsp.readFile(file, 'utf8')); } catch { return null; }
|
|
93
|
+
},
|
|
94
|
+
async save(entries) {
|
|
95
|
+
await fsp.writeFile(file, JSON.stringify({ builtAt: Date.now(), entries }));
|
|
96
|
+
},
|
|
97
|
+
async close({ keep } = {}) {
|
|
98
|
+
if (!persist && !keep) { try { await fsp.unlink(file); } catch { /* gone */ } }
|
|
99
|
+
},
|
|
100
|
+
describe() { return `superlite/json (${persist ? 'persisted' : 'ephemeral'}) ${file}`; }
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// sqlite-backed store at an explicit file. cleanup=true removes the file on close.
|
|
105
|
+
function sqliteAt(file, { cleanup }) {
|
|
106
|
+
let db = null;
|
|
107
|
+
const open = async () => {
|
|
108
|
+
if (db) return db;
|
|
109
|
+
const { DatabaseSync } = await import('node:sqlite');
|
|
110
|
+
db = new DatabaseSync(file);
|
|
111
|
+
db.exec('CREATE TABLE IF NOT EXISTS files (path TEXT PRIMARY KEY, name TEXT, ext TEXT, mtime REAL, content TEXT)');
|
|
112
|
+
return db;
|
|
113
|
+
};
|
|
114
|
+
return {
|
|
115
|
+
kind: 'lite',
|
|
116
|
+
file,
|
|
117
|
+
async load() {
|
|
118
|
+
const d = await open();
|
|
119
|
+
const rows = d.prepare('SELECT path, name, ext, mtime, content FROM files').all();
|
|
120
|
+
return rows.length ? { entries: rows } : null;
|
|
121
|
+
},
|
|
122
|
+
async save(entries) {
|
|
123
|
+
const d = await open();
|
|
124
|
+
d.exec('BEGIN');
|
|
125
|
+
d.exec('DELETE FROM files');
|
|
126
|
+
const ins = d.prepare('INSERT OR REPLACE INTO files (path, name, ext, mtime, content) VALUES (?, ?, ?, ?, ?)');
|
|
127
|
+
for (const e of entries) ins.run(e.path, e.name, e.ext, e.mtime, e.content);
|
|
128
|
+
d.exec('COMMIT');
|
|
129
|
+
},
|
|
130
|
+
async close({ keep } = {}) {
|
|
131
|
+
if (db) { db.close(); db = null; }
|
|
132
|
+
if (cleanup && !keep) { try { await fsp.unlink(file); } catch { /* gone */ } }
|
|
133
|
+
},
|
|
134
|
+
describe() { return `lite/sqlite (${cleanup ? 'ephemeral' : 'persisted'}) ${file}`; }
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// custom: a connection URL the user supplies (--store-url / DATABASE_URL).
|
|
139
|
+
function customStore(url) {
|
|
140
|
+
if (!url) throw new Error('custom store needs a connection URL (--store-url or DATABASE_URL)');
|
|
141
|
+
let proto;
|
|
142
|
+
try { proto = new URL(url).protocol.replace(':', ''); } catch { throw new Error(`invalid store URL: ${url}`); }
|
|
143
|
+
|
|
144
|
+
if (proto === 'sqlite' || proto === 'file') {
|
|
145
|
+
const f = url.replace(/^sqlite:\/\/|^file:\/\/|^sqlite:/, '') || ':memory:';
|
|
146
|
+
return sqliteAt(path.resolve(f), { cleanup: false }); // the user's own file - never delete
|
|
147
|
+
}
|
|
148
|
+
if (proto === 'postgres' || proto === 'postgresql') return postgresStore(url);
|
|
149
|
+
throw new Error(`unsupported store URL scheme "${proto}://" - use sqlite:// or postgres://`);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function postgresStore(url) {
|
|
153
|
+
let client = null;
|
|
154
|
+
const open = async () => {
|
|
155
|
+
if (client) return client;
|
|
156
|
+
let pg;
|
|
157
|
+
try { pg = (await import('pg')).default; }
|
|
158
|
+
catch { throw new Error('the postgres store needs the "pg" package - run: npm i pg'); }
|
|
159
|
+
client = new pg.Client({ connectionString: url });
|
|
160
|
+
await client.connect();
|
|
161
|
+
await client.query(
|
|
162
|
+
'CREATE TABLE IF NOT EXISTS projview_files (path TEXT PRIMARY KEY, name TEXT, ext TEXT, mtime DOUBLE PRECISION, content TEXT)'
|
|
163
|
+
);
|
|
164
|
+
return client;
|
|
165
|
+
};
|
|
166
|
+
return {
|
|
167
|
+
kind: 'custom',
|
|
168
|
+
async load() {
|
|
169
|
+
const c = await open();
|
|
170
|
+
const { rows } = await c.query('SELECT path, name, ext, mtime, content FROM projview_files');
|
|
171
|
+
return rows.length ? { entries: rows } : null;
|
|
172
|
+
},
|
|
173
|
+
async save(entries) {
|
|
174
|
+
const c = await open();
|
|
175
|
+
await c.query('BEGIN');
|
|
176
|
+
await c.query('DELETE FROM projview_files');
|
|
177
|
+
for (const e of entries) {
|
|
178
|
+
await c.query(
|
|
179
|
+
'INSERT INTO projview_files (path, name, ext, mtime, content) VALUES ($1, $2, $3, $4, $5)' +
|
|
180
|
+
' ON CONFLICT (path) DO UPDATE SET name = $2, ext = $3, mtime = $4, content = $5',
|
|
181
|
+
[e.path, e.name, e.ext, e.mtime, e.content]
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
await c.query('COMMIT');
|
|
185
|
+
},
|
|
186
|
+
async close() { if (client) { await client.end(); client = null; } }, // never drop the user's table
|
|
187
|
+
describe() { return `custom/postgres ${redact(url)}`; }
|
|
188
|
+
};
|
|
189
|
+
}
|