tinywebui 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -13
- package/bin/tinywebui.js +69 -63
- package/package.json +25 -8
- package/public/app.js +40 -14
- package/public/chat/composer.js +1 -1
- package/public/chat/sidebar.js +8 -5
- package/public/index.html +88 -49
- package/public/panels/admin.js +40 -1
- package/public/panels/model-editor.js +336 -0
- package/public/panels/panels.js +37 -12
- package/public/panels/settings-home.js +125 -0
- package/public/panels/settings.js +124 -36
- package/public/shared/attachment_note.js +7 -1
- package/public/shared/events.js +1 -0
- package/public/shared/features.js +1 -0
- package/public/shared/origins.js +1 -0
- package/public/shared/usage.js +16 -1
- package/public/styles.css +83 -20
- package/src/access/auth.js +155 -155
- package/src/access/auth_gate.js +39 -38
- package/src/access/email.js +17 -0
- package/src/access/policy.js +123 -138
- package/src/audit.js +1 -1
- package/src/automations/automation.js +135 -123
- package/src/automations/scheduler.js +138 -127
- package/src/chat/compact.js +149 -156
- package/src/chat/llm.js +380 -382
- package/src/chat/runs.js +162 -151
- package/src/chat/tool_executor.js +229 -218
- package/src/cli.js +535 -433
- package/src/config/approval.js +35 -29
- package/src/config/config.js +546 -451
- package/src/config/data_dir.js +24 -0
- package/src/config/models.js +191 -115
- package/src/config/refs.js +32 -0
- package/src/config/secrets.js +153 -0
- package/src/files/documents.js +57 -68
- package/src/files/images.js +30 -31
- package/src/harness/agent.js +60 -60
- package/src/harness/attribution.js +66 -58
- package/src/harness/checkpoint.js +67 -62
- package/src/harness/history.js +28 -23
- package/src/harness/prompt.js +40 -57
- package/src/harness/provider.js +284 -283
- package/src/harness/types.js +1 -0
- package/src/http.js +98 -66
- package/src/log.js +11 -12
- package/src/mcp.js +398 -374
- package/src/retrieval/embedding.js +228 -240
- package/src/retrieval/retrieval.js +182 -173
- package/src/routes/admin.js +178 -138
- package/src/routes/auth.js +184 -100
- package/src/routes/automations.js +99 -82
- package/src/routes/chats.js +240 -218
- package/src/routes/documents.js +130 -120
- package/src/routes/library.js +43 -46
- package/src/routes/mcp.js +49 -49
- package/src/routes/settings.js +127 -122
- package/src/routes/types.js +1 -0
- package/src/server.js +428 -392
- package/src/store/automations.js +106 -106
- package/src/store/chats.js +217 -242
- package/src/store/documents.js +181 -185
- package/src/store/embeddings.js +52 -54
- package/src/store/index.js +95 -49
- package/src/store/messages.js +55 -80
- package/src/store/schema.js +128 -107
- package/src/store/scope.js +18 -17
- package/src/store/search.js +25 -27
- package/src/store/turns.js +53 -53
- package/src/store/usage.js +343 -269
- package/src/store/users.js +162 -87
- package/src/store/wire.js +50 -41
- package/src/tools/ask_tool.js +68 -68
- package/src/tools/chat_search_tool.js +38 -44
- package/src/tools/context_tool.js +131 -138
- package/src/tools/document_tool.js +73 -78
- package/src/tools/mcp_tool.js +274 -232
- package/src/tools/task_tool.js +53 -50
- package/src/tools/types.js +1 -0
- package/public/panels/connectors.js +0 -184
- package/src/set-password.js +0 -43
package/src/store/scope.js
CHANGED
|
@@ -1,28 +1,28 @@
|
|
|
1
1
|
/* Helpers every area of the store shares: whose rows a query sees, and safe FTS5 queries. */
|
|
2
|
-
|
|
3
2
|
/**
|
|
4
3
|
* Every read of user-owned rows names whose rows it wants. There is no
|
|
5
4
|
* default: `undefined` throws, so a caller that forgot to pass the
|
|
6
5
|
* authenticated user fails loudly instead of silently seeing everyone's data.
|
|
7
6
|
* string that user's rows
|
|
8
7
|
* null rows with no owner (legacy / pre-auth rows)
|
|
9
|
-
* ALL_USERS no filter -- authMode '
|
|
8
|
+
* ALL_USERS no filter -- authMode 'local', admin views, internal plumbing
|
|
10
9
|
*/
|
|
11
10
|
export const ALL_USERS = Symbol('all-users');
|
|
12
|
-
|
|
13
11
|
export function scope(userId, col = 'user_id') {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
12
|
+
if (userId === ALL_USERS)
|
|
13
|
+
return { sql: '1=1', params: [] };
|
|
14
|
+
if (userId === null)
|
|
15
|
+
return { sql: `${col} IS NULL`, params: [] };
|
|
16
|
+
if (typeof userId === 'string' && userId)
|
|
17
|
+
return { sql: `${col} = ?`, params: [userId] };
|
|
18
|
+
throw new TypeError('store: a user scope is required (pass a user id, null, or ALL_USERS)');
|
|
18
19
|
}
|
|
19
|
-
|
|
20
20
|
/** The owner to write onto a new row: ALL_USERS (no auth) owns as null. */
|
|
21
21
|
export function ownerOf(userId) {
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
if (userId === ALL_USERS || userId === null || userId === undefined)
|
|
23
|
+
return null;
|
|
24
|
+
return String(userId);
|
|
24
25
|
}
|
|
25
|
-
|
|
26
26
|
/**
|
|
27
27
|
* Turns free text from a search box into an FTS5 MATCH expression that cannot
|
|
28
28
|
* fail to parse. FTS5's own query grammar has ANDs, ORs, dashes, colons and
|
|
@@ -34,10 +34,11 @@ export function ownerOf(userId) {
|
|
|
34
34
|
* a whole word is finished.
|
|
35
35
|
*/
|
|
36
36
|
export function ftsQuery(raw, { any = false } = {}) {
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
37
|
+
const tokens = String(raw ?? '').trim().split(/\s+/).filter(Boolean).slice(0, 12);
|
|
38
|
+
if (!tokens.length)
|
|
39
|
+
return '';
|
|
40
|
+
const esc = (t) => t.replace(/"/g, '""');
|
|
41
|
+
return tokens
|
|
42
|
+
.map((t, i) => (i === tokens.length - 1 ? `"${esc(t)}"*` : `"${esc(t)}"`))
|
|
43
|
+
.join(any ? ' OR ' : ' ');
|
|
43
44
|
}
|
package/src/store/search.js
CHANGED
|
@@ -1,33 +1,31 @@
|
|
|
1
1
|
import { scope, ftsQuery } from './scope.js';
|
|
2
|
-
|
|
3
2
|
// Sidebar search: full-text over what was said, one row per chat.
|
|
4
3
|
// One area of the Store; see index.js.
|
|
5
|
-
|
|
6
4
|
export class SearchStore {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
5
|
+
db;
|
|
6
|
+
constructor(db) {
|
|
7
|
+
this.db = db;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Full-text search, one row per matching CHAT, not per matching message --
|
|
11
|
+
* a search box is for finding a conversation, and a chat where the same
|
|
12
|
+
* word landed in five messages should not crowd out four other chats that
|
|
13
|
+
* only said it once. Each chat's own best-ranked message stands for it,
|
|
14
|
+
* and chats are then ordered against each other by that same rank.
|
|
15
|
+
*
|
|
16
|
+
* `bm25()` and `snippet()` are FTS5 auxiliary functions: usable only in a
|
|
17
|
+
* query that itself matches the virtual table, not through an arbitrary
|
|
18
|
+
* subquery. Computing both once in `hits` and carrying the values through
|
|
19
|
+
* `ranked` (rather than recomputing `bm25(messages_fts)` inside the window
|
|
20
|
+
* function's own ORDER BY) is what keeps them legal here -- SQLite raises
|
|
21
|
+
* "unable to use function bm25 in the requested context" otherwise.
|
|
22
|
+
*/
|
|
23
|
+
chats(query, limit, userId) {
|
|
24
|
+
const s = scope(userId, 'c.user_id');
|
|
25
|
+
const q = ftsQuery(query);
|
|
26
|
+
if (!q)
|
|
27
|
+
return [];
|
|
28
|
+
return this.db.prepare(`
|
|
31
29
|
WITH hits AS (
|
|
32
30
|
SELECT
|
|
33
31
|
m.chat_id AS chatId,
|
|
@@ -52,5 +50,5 @@ export class SearchStore {
|
|
|
52
50
|
ORDER BY rank
|
|
53
51
|
LIMIT ?
|
|
54
52
|
`).all(q, ...s.params, limit);
|
|
55
|
-
|
|
53
|
+
}
|
|
56
54
|
}
|
package/src/store/turns.js
CHANGED
|
@@ -1,78 +1,78 @@
|
|
|
1
1
|
import { scope, ftsQuery } from './scope.js';
|
|
2
2
|
import { CORPUS } from './embeddings.js';
|
|
3
|
-
|
|
4
3
|
// Chat turns -- a question and its final answer -- the 'chats' retrieval corpus.
|
|
5
4
|
// One area of the Store; see index.js.
|
|
6
|
-
|
|
7
5
|
export class TurnStore {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
const rows = this.db.prepare(`
|
|
6
|
+
constructor(db, deps) {
|
|
7
|
+
this.db = db;
|
|
8
|
+
Object.assign(this, deps);
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* A chat's turns: each question with the answer it finally got -- the last
|
|
12
|
+
* assistant message with text before the next question, the one the
|
|
13
|
+
* transcript shows as the answer. A turn's id is its user message's id.
|
|
14
|
+
*/
|
|
15
|
+
forChat(chatId) {
|
|
16
|
+
const rows = this.db.prepare(`
|
|
20
17
|
SELECT id, seq, role, content, created_at FROM messages
|
|
21
18
|
WHERE chat_id = ? AND role IN ('user', 'assistant') ORDER BY seq
|
|
22
19
|
`).all(chatId);
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
20
|
+
const turns = [];
|
|
21
|
+
for (const r of rows) {
|
|
22
|
+
if (r.role === 'user')
|
|
23
|
+
turns.push({ id: r.id, chatId, seq: r.seq, question: r.content || '', answer: '', createdAt: r.created_at });
|
|
24
|
+
else if (turns.length && r.content?.trim())
|
|
25
|
+
turns.at(-1).answer = r.content;
|
|
26
|
+
}
|
|
27
|
+
return turns;
|
|
27
28
|
}
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
/** The turns with these ids, each with its chat's title: Map(id -> turn). */
|
|
32
|
-
byIds(ids) {
|
|
33
|
-
const chatIds = this.db.prepare(`
|
|
29
|
+
/** The turns with these ids, each with its chat's title: Map(id -> turn). */
|
|
30
|
+
byIds(ids) {
|
|
31
|
+
const chatIds = this.db.prepare(`
|
|
34
32
|
SELECT DISTINCT chat_id FROM messages WHERE role = 'user' AND id IN (SELECT value FROM json_each(?))
|
|
35
33
|
`).all(JSON.stringify(ids)).map((r) => r.chat_id);
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
34
|
+
const wanted = new Set(ids);
|
|
35
|
+
const out = new Map();
|
|
36
|
+
for (const chatId of chatIds) {
|
|
37
|
+
const title = this.chats.byId(chatId)?.title || 'Untitled chat';
|
|
38
|
+
for (const t of this.forChat(chatId))
|
|
39
|
+
if (wanted.has(t.id))
|
|
40
|
+
out.set(t.id, { ...t, chatTitle: title });
|
|
41
|
+
}
|
|
42
|
+
return out;
|
|
41
43
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
ids(userId, { excludeChatId = null } = {}) {
|
|
47
|
-
const s = scope(userId, 'c.user_id');
|
|
48
|
-
return this.db.prepare(`
|
|
44
|
+
/** Every turn id a user may search, leaving one chat out (the one asking). */
|
|
45
|
+
ids(userId, { excludeChatId = null } = {}) {
|
|
46
|
+
const s = scope(userId, 'c.user_id');
|
|
47
|
+
return this.db.prepare(`
|
|
49
48
|
SELECT m.id FROM messages m JOIN chats c ON c.id = m.chat_id
|
|
50
49
|
WHERE m.role = 'user' AND m.chat_id IS NOT ? AND ${s.sql}
|
|
51
50
|
`).all(excludeChatId, ...s.params).map((r) => r.id);
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
51
|
+
}
|
|
52
|
+
/** BM25 per turn (its best message), higher is better. `any`: any query word may match. */
|
|
53
|
+
lexicalScores(userId, query, { excludeChatId = null, any = true } = {}) {
|
|
54
|
+
const q = ftsQuery(query, { any });
|
|
55
|
+
if (!q)
|
|
56
|
+
return new Map();
|
|
57
|
+
const s = scope(userId, 'c.user_id');
|
|
58
|
+
const rows = this.db.prepare(`
|
|
60
59
|
SELECT bm25(messages_fts) AS rank,
|
|
61
60
|
(SELECT u.id FROM messages u WHERE u.chat_id = m.chat_id AND u.role = 'user' AND u.seq <= m.seq ORDER BY u.seq DESC LIMIT 1) AS turn
|
|
62
61
|
FROM messages_fts JOIN messages m ON m.id = messages_fts.rowid JOIN chats c ON c.id = m.chat_id
|
|
63
62
|
WHERE messages_fts MATCH ? AND m.chat_id IS NOT ? AND ${s.sql}
|
|
64
63
|
`).all(q, excludeChatId, ...s.params);
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
64
|
+
const out = new Map();
|
|
65
|
+
for (const r of rows)
|
|
66
|
+
if (r.turn != null)
|
|
67
|
+
out.set(r.turn, Math.max(out.get(r.turn) ?? -Infinity, -r.rank));
|
|
68
|
+
return out;
|
|
69
|
+
}
|
|
70
|
+
/** Chats with a question not yet embedded under this model (for backfill). */
|
|
71
|
+
chatsMissingVectors(modelKey) {
|
|
72
|
+
return this.db.prepare(`
|
|
73
73
|
SELECT DISTINCT m.chat_id AS id FROM messages m
|
|
74
74
|
LEFT JOIN embeddings e ON e.corpus = ? AND e.unit_id = m.id AND e.model_key = ? AND e.chunk = 0
|
|
75
75
|
WHERE m.role = 'user' AND e.unit_id IS NULL
|
|
76
76
|
`).all(CORPUS.CHATS, modelKey).map((r) => r.id);
|
|
77
|
-
|
|
77
|
+
}
|
|
78
78
|
}
|