persnally 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +110 -0
- package/README.md +96 -0
- package/build/src/cli.d.ts +6 -0
- package/build/src/cli.js +404 -0
- package/build/src/config.d.ts +10 -0
- package/build/src/config.js +42 -0
- package/build/src/connect.d.ts +13 -0
- package/build/src/connect.js +51 -0
- package/build/src/consolidate.d.ts +18 -0
- package/build/src/consolidate.js +67 -0
- package/build/src/daemon.d.ts +9 -0
- package/build/src/daemon.js +167 -0
- package/build/src/dashboard.html +181 -0
- package/build/src/decay.d.ts +19 -0
- package/build/src/decay.js +33 -0
- package/build/src/events.d.ts +180 -0
- package/build/src/events.js +133 -0
- package/build/src/importers/chatgpt.d.ts +9 -0
- package/build/src/importers/chatgpt.js +34 -0
- package/build/src/importers/claude-code.d.ts +16 -0
- package/build/src/importers/claude-code.js +99 -0
- package/build/src/importers/claude.d.ts +8 -0
- package/build/src/importers/claude.js +52 -0
- package/build/src/importers/extract.d.ts +31 -0
- package/build/src/importers/extract.js +53 -0
- package/build/src/importers/git.d.ts +23 -0
- package/build/src/importers/git.js +123 -0
- package/build/src/lifecycle.d.ts +14 -0
- package/build/src/lifecycle.js +119 -0
- package/build/src/llm.d.ts +25 -0
- package/build/src/llm.js +76 -0
- package/build/src/mcp/daemon-client.d.ts +11 -0
- package/build/src/mcp/daemon-client.js +42 -0
- package/build/src/mcp/index.d.ts +10 -0
- package/build/src/mcp/index.js +158 -0
- package/build/src/mcp/migrate-v1.d.ts +6 -0
- package/build/src/mcp/migrate-v1.js +48 -0
- package/build/src/mcp/telemetry.d.ts +8 -0
- package/build/src/mcp/telemetry.js +29 -0
- package/build/src/paths.d.ts +2 -0
- package/build/src/paths.js +4 -0
- package/build/src/permissions.d.ts +14 -0
- package/build/src/permissions.js +33 -0
- package/build/src/profile.d.ts +22 -0
- package/build/src/profile.js +62 -0
- package/build/src/setup.d.ts +23 -0
- package/build/src/setup.js +111 -0
- package/build/src/store.d.ts +62 -0
- package/build/src/store.js +233 -0
- package/package.json +56 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local-only telemetry for the Phase 0 capture-rate experiment.
|
|
3
|
+
* Appends one JSON line per event to ~/.persnally/telemetry.jsonl — counts and
|
|
4
|
+
* timestamps only, never conversation content. Analyzed by experiments/capture_rate.py.
|
|
5
|
+
*/
|
|
6
|
+
import { appendFileSync, existsSync, mkdirSync } from "fs";
|
|
7
|
+
import { join } from "path";
|
|
8
|
+
import { homedir } from "os";
|
|
9
|
+
const DIR = join(homedir(), ".persnally");
|
|
10
|
+
const FILE = join(DIR, "telemetry.jsonl");
|
|
11
|
+
let clientName = "unknown";
|
|
12
|
+
export function setClient(name) {
|
|
13
|
+
if (name)
|
|
14
|
+
clientName = name;
|
|
15
|
+
}
|
|
16
|
+
export function getClient() {
|
|
17
|
+
return clientName;
|
|
18
|
+
}
|
|
19
|
+
export function logEvent(event, data = {}) {
|
|
20
|
+
try {
|
|
21
|
+
if (!existsSync(DIR))
|
|
22
|
+
mkdirSync(DIR, { recursive: true });
|
|
23
|
+
const line = JSON.stringify({ ts: new Date().toISOString(), event, client: clientName, ...data });
|
|
24
|
+
appendFileSync(FILE, line + "\n");
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
// Telemetry must never break the server.
|
|
28
|
+
}
|
|
29
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-client category scopes. Default-open: a client with no entry sees
|
|
3
|
+
* everything. Once scoped, it sees only its allowed categories — enforced
|
|
4
|
+
* at the daemon, so no MCP client can read past its grant. Stored in config.
|
|
5
|
+
*/
|
|
6
|
+
export declare const CATEGORIES: readonly ["technology", "business", "finance", "career", "health", "science", "creative", "education", "lifestyle", "news", "other"];
|
|
7
|
+
export type Category = (typeof CATEGORIES)[number];
|
|
8
|
+
export type Scopes = Record<string, Category[]>;
|
|
9
|
+
export declare function loadScopes(): Scopes;
|
|
10
|
+
export declare function setScope(client: string, categories: Category[]): void;
|
|
11
|
+
export declare function clearScope(client: string): boolean;
|
|
12
|
+
/** null = unrestricted (sees all). An array = the only categories this client may read. */
|
|
13
|
+
export declare function allowedCategories(client: string): Category[] | null;
|
|
14
|
+
export declare function isAllowed(client: string, category: string): boolean;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-client category scopes. Default-open: a client with no entry sees
|
|
3
|
+
* everything. Once scoped, it sees only its allowed categories — enforced
|
|
4
|
+
* at the daemon, so no MCP client can read past its grant. Stored in config.
|
|
5
|
+
*/
|
|
6
|
+
import { loadConfig, saveConfig } from "./config.js";
|
|
7
|
+
export const CATEGORIES = [
|
|
8
|
+
"technology", "business", "finance", "career", "health",
|
|
9
|
+
"science", "creative", "education", "lifestyle", "news", "other",
|
|
10
|
+
];
|
|
11
|
+
export function loadScopes() {
|
|
12
|
+
const s = loadConfig().client_scopes;
|
|
13
|
+
return s && typeof s === "object" ? s : {};
|
|
14
|
+
}
|
|
15
|
+
export function setScope(client, categories) {
|
|
16
|
+
saveConfig({ client_scopes: { ...loadScopes(), [client]: categories } });
|
|
17
|
+
}
|
|
18
|
+
export function clearScope(client) {
|
|
19
|
+
const scopes = loadScopes();
|
|
20
|
+
if (!(client in scopes))
|
|
21
|
+
return false;
|
|
22
|
+
delete scopes[client];
|
|
23
|
+
saveConfig({ client_scopes: scopes });
|
|
24
|
+
return true;
|
|
25
|
+
}
|
|
26
|
+
/** null = unrestricted (sees all). An array = the only categories this client may read. */
|
|
27
|
+
export function allowedCategories(client) {
|
|
28
|
+
return loadScopes()[client] ?? null;
|
|
29
|
+
}
|
|
30
|
+
export function isAllowed(client, category) {
|
|
31
|
+
const allowed = allowedCategories(client);
|
|
32
|
+
return allowed === null || allowed.includes(category);
|
|
33
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Profile synthesis — the Mirror. Turns the event store into a descriptive profile,
|
|
3
|
+
* each section citing the event ids it rests on so "why does it think this?"
|
|
4
|
+
* resolves to actual evidence.
|
|
5
|
+
*/
|
|
6
|
+
import { z } from "zod";
|
|
7
|
+
import { type LlmExtract } from "./llm.js";
|
|
8
|
+
import type { EventStore } from "./store.js";
|
|
9
|
+
export declare const profileSchema: z.ZodObject<{
|
|
10
|
+
headline: z.ZodString;
|
|
11
|
+
sections: z.ZodArray<z.ZodObject<{
|
|
12
|
+
title: z.ZodString;
|
|
13
|
+
body: z.ZodString;
|
|
14
|
+
evidence_event_ids: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
15
|
+
}, z.core.$strip>>;
|
|
16
|
+
}, z.core.$strip>;
|
|
17
|
+
export type Profile = z.infer<typeof profileSchema> & {
|
|
18
|
+
generated_at: string;
|
|
19
|
+
model: string;
|
|
20
|
+
};
|
|
21
|
+
export declare function synthesizeProfile(store: EventStore, extract?: LlmExtract, model?: string): Promise<Profile>;
|
|
22
|
+
export declare function renderProfile(p: Profile): string;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Profile synthesis — the Mirror. Turns the event store into a descriptive profile,
|
|
3
|
+
* each section citing the event ids it rests on so "why does it think this?"
|
|
4
|
+
* resolves to actual evidence.
|
|
5
|
+
*/
|
|
6
|
+
import { z } from "zod";
|
|
7
|
+
import { anthropicExtract, DEFAULT_PROFILE_MODEL } from "./llm.js";
|
|
8
|
+
export const profileSchema = z.object({
|
|
9
|
+
headline: z.string().min(1),
|
|
10
|
+
sections: z.array(z.object({
|
|
11
|
+
title: z.string().min(1),
|
|
12
|
+
body: z.string().min(1),
|
|
13
|
+
evidence_event_ids: z.array(z.string()).default([]),
|
|
14
|
+
})).min(1),
|
|
15
|
+
});
|
|
16
|
+
const INSTRUCTION = `Write the sharpest possible picture of this person from their extracted signals.
|
|
17
|
+
|
|
18
|
+
Rules:
|
|
19
|
+
- Cover only what the evidence supports: current work, how they think and decide, technical depth, communication style, what they care about and avoid, and non-obvious inferences the pattern reveals.
|
|
20
|
+
- Be specific and concrete. Where you're inferring rather than told, say so. Do not flatter.
|
|
21
|
+
- Every section must list the event ids (given in [brackets]) of the signals it rests on.
|
|
22
|
+
- The test: the person reads it and thinks "how did it know that?"`;
|
|
23
|
+
export async function synthesizeProfile(store, extract = anthropicExtract, model = DEFAULT_PROFILE_MODEL) {
|
|
24
|
+
const topics = store.topics(30);
|
|
25
|
+
const assertions = store.query({ type: "signal.assertion", limit: 200 });
|
|
26
|
+
if (!topics.length && !assertions.length) {
|
|
27
|
+
throw new Error("Nothing to synthesize from — run an import first.");
|
|
28
|
+
}
|
|
29
|
+
const content = [
|
|
30
|
+
"## Weighted interests (decayed)",
|
|
31
|
+
...topics.map((t) => `- [${t.event_ids[0] ?? ""}] ${t.topic} (${t.category}, weight ${t.weight.toFixed(2)}, ` +
|
|
32
|
+
`${t.dominant_intent}, ${t.signals} signals${t.entities.length ? `, entities: ${t.entities.slice(0, 5).join(", ")}` : ""})`),
|
|
33
|
+
"",
|
|
34
|
+
"## Extracted assertions",
|
|
35
|
+
...assertions.map((e) => {
|
|
36
|
+
const p = e.payload;
|
|
37
|
+
return `- [${e.id}] (${p.kind}, conf ${p.confidence}) ${p.claim} — ${p.evidence}`;
|
|
38
|
+
}),
|
|
39
|
+
].join("\n");
|
|
40
|
+
const raw = await extract({
|
|
41
|
+
model,
|
|
42
|
+
instruction: INSTRUCTION,
|
|
43
|
+
schema: profileSchema,
|
|
44
|
+
content,
|
|
45
|
+
maxTokens: 8000,
|
|
46
|
+
});
|
|
47
|
+
const parsed = profileSchema.parse(raw);
|
|
48
|
+
const profile = { ...parsed, generated_at: new Date().toISOString(), model };
|
|
49
|
+
store.saveProfile(profile);
|
|
50
|
+
return profile;
|
|
51
|
+
}
|
|
52
|
+
export function renderProfile(p) {
|
|
53
|
+
const lines = [`# ${p.headline}`, ""];
|
|
54
|
+
for (const s of p.sections) {
|
|
55
|
+
lines.push(`## ${s.title}`, s.body);
|
|
56
|
+
if (s.evidence_event_ids.length)
|
|
57
|
+
lines.push(` ↳ evidence: ${s.evidence_event_ids.length} event(s)`);
|
|
58
|
+
lines.push("");
|
|
59
|
+
}
|
|
60
|
+
lines.push(`(generated ${p.generated_at} by ${p.model})`);
|
|
61
|
+
return lines.join("\n");
|
|
62
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One-command onboarding: find exports (zipped or unzipped), pick an engine,
|
|
3
|
+
* import everything, synthesize, connect AI clients, open the dashboard.
|
|
4
|
+
* Idempotent — already-imported sources are recorded in config and skipped.
|
|
5
|
+
*/
|
|
6
|
+
export type ExportKind = "claude" | "chatgpt";
|
|
7
|
+
export interface FoundExport {
|
|
8
|
+
kind: ExportKind;
|
|
9
|
+
path: string;
|
|
10
|
+
origin: string;
|
|
11
|
+
cleanup?: string;
|
|
12
|
+
}
|
|
13
|
+
/** Scans a directory (default ~/Downloads) for Claude/ChatGPT exports, zipped or not. */
|
|
14
|
+
export declare function detectExports(searchDir?: string): FoundExport[];
|
|
15
|
+
import { type PersnallyEvent } from "./events.js";
|
|
16
|
+
import type { ChosenExtractor } from "./llm.js";
|
|
17
|
+
export declare const DENSITY_QUESTIONS: readonly ["What are you working on right now?", "What topics or technologies do you care most about these days?"];
|
|
18
|
+
export declare function isThin(signalCount: number): boolean;
|
|
19
|
+
/** Turns free-text answers into seed events — via the engine when available,
|
|
20
|
+
* else a deterministic phrase split so the key-free path still works. */
|
|
21
|
+
export declare function eventsFromAnswers(answers: string[], engine: ChosenExtractor | null): Promise<PersnallyEvent[]>;
|
|
22
|
+
export declare function alreadyImported(origin: string): boolean;
|
|
23
|
+
export declare function markImported(origin: string): void;
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One-command onboarding: find exports (zipped or unzipped), pick an engine,
|
|
3
|
+
* import everything, synthesize, connect AI clients, open the dashboard.
|
|
4
|
+
* Idempotent — already-imported sources are recorded in config and skipped.
|
|
5
|
+
*/
|
|
6
|
+
import { execFileSync } from "node:child_process";
|
|
7
|
+
import { existsSync, mkdtempSync, readdirSync, rmSync, openSync, readSync, closeSync } from "node:fs";
|
|
8
|
+
import { homedir, tmpdir } from "node:os";
|
|
9
|
+
import { join } from "node:path";
|
|
10
|
+
import { loadConfig, saveConfig } from "./config.js";
|
|
11
|
+
function sniffKind(conversationsJson) {
|
|
12
|
+
const fd = openSync(conversationsJson, "r");
|
|
13
|
+
const buf = Buffer.alloc(4096);
|
|
14
|
+
const n = readSync(fd, buf, 0, buf.length, 0);
|
|
15
|
+
closeSync(fd);
|
|
16
|
+
const head = buf.toString("utf-8", 0, n);
|
|
17
|
+
if (head.includes('"chat_messages"'))
|
|
18
|
+
return "claude";
|
|
19
|
+
if (head.includes('"mapping"'))
|
|
20
|
+
return "chatgpt";
|
|
21
|
+
return null;
|
|
22
|
+
}
|
|
23
|
+
function zipHasConversations(zipPath) {
|
|
24
|
+
try {
|
|
25
|
+
return execFileSync("unzip", ["-l", zipPath], { encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"] })
|
|
26
|
+
.includes("conversations.json");
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
/** Scans a directory (default ~/Downloads) for Claude/ChatGPT exports, zipped or not. */
|
|
33
|
+
export function detectExports(searchDir = join(homedir(), "Downloads")) {
|
|
34
|
+
if (!existsSync(searchDir))
|
|
35
|
+
return [];
|
|
36
|
+
const found = [];
|
|
37
|
+
for (const entry of readdirSync(searchDir, { withFileTypes: true })) {
|
|
38
|
+
const full = join(searchDir, entry.name);
|
|
39
|
+
if (entry.isDirectory() && existsSync(join(full, "conversations.json"))) {
|
|
40
|
+
const kind = sniffKind(join(full, "conversations.json"));
|
|
41
|
+
if (kind)
|
|
42
|
+
found.push({ kind, path: full, origin: full });
|
|
43
|
+
}
|
|
44
|
+
else if (entry.isFile() && entry.name.endsWith(".zip") && zipHasConversations(full)) {
|
|
45
|
+
const tmp = mkdtempSync(join(tmpdir(), "persnally-export-"));
|
|
46
|
+
execFileSync("unzip", ["-q", full, "-d", tmp]);
|
|
47
|
+
const root = existsSync(join(tmp, "conversations.json"))
|
|
48
|
+
? tmp
|
|
49
|
+
: readdirSync(tmp).map((d) => join(tmp, d)).find((d) => existsSync(join(d, "conversations.json")));
|
|
50
|
+
const kind = root ? sniffKind(join(root, "conversations.json")) : null;
|
|
51
|
+
if (root && kind)
|
|
52
|
+
found.push({ kind, path: root, origin: full, cleanup: tmp });
|
|
53
|
+
else
|
|
54
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return found;
|
|
58
|
+
}
|
|
59
|
+
// ── Density floor: never leave the user with an empty mirror ──
|
|
60
|
+
import { z } from "zod";
|
|
61
|
+
import { newEvent, PAYLOAD_SCHEMAS } from "./events.js";
|
|
62
|
+
const THIN_SIGNAL_THRESHOLD = 15;
|
|
63
|
+
export const DENSITY_QUESTIONS = [
|
|
64
|
+
"What are you working on right now?",
|
|
65
|
+
"What topics or technologies do you care most about these days?",
|
|
66
|
+
];
|
|
67
|
+
export function isThin(signalCount) {
|
|
68
|
+
return signalCount < THIN_SIGNAL_THRESHOLD;
|
|
69
|
+
}
|
|
70
|
+
const seedExtraction = z.object({ topics: z.array(PAYLOAD_SCHEMAS["signal.topic"]) });
|
|
71
|
+
/** Turns free-text answers into seed events — via the engine when available,
|
|
72
|
+
* else a deterministic phrase split so the key-free path still works. */
|
|
73
|
+
export async function eventsFromAnswers(answers, engine) {
|
|
74
|
+
if (!answers.join("").replace(/[^a-zA-Z0-9]/g, ""))
|
|
75
|
+
return [];
|
|
76
|
+
const text = answers.map((a, i) => `${DENSITY_QUESTIONS[i] ?? "Q"}: ${a}`).join("\n");
|
|
77
|
+
if (engine) {
|
|
78
|
+
const result = await engine.extract({
|
|
79
|
+
model: engine.model,
|
|
80
|
+
instruction: "The user answered two onboarding questions about themselves. Extract 2-6 topic signals. Weight by how central each seems; intent 'building' for active work, 'learning'/'researching' for interests.",
|
|
81
|
+
schema: seedExtraction,
|
|
82
|
+
content: text,
|
|
83
|
+
});
|
|
84
|
+
return seedExtraction.parse(result).topics.map((t) => newEvent("signal.topic", "cli", t, { kind: "local", surface: "cli" }));
|
|
85
|
+
}
|
|
86
|
+
// Key-free fallback: comma/and-separated phrases become moderate signals.
|
|
87
|
+
const phrases = answers
|
|
88
|
+
.flatMap((a) => a.split(/,|\band\b|;/))
|
|
89
|
+
.map((p) => p.trim())
|
|
90
|
+
.filter((p) => p.length > 2 && p.length < 80 && /[a-zA-Z0-9]/.test(p))
|
|
91
|
+
.slice(0, 8);
|
|
92
|
+
return phrases.map((topic, i) => newEvent("signal.topic", "cli", {
|
|
93
|
+
topic,
|
|
94
|
+
weight: 0.6,
|
|
95
|
+
intent: i === 0 ? "building" : "discussing",
|
|
96
|
+
sentiment: "neutral",
|
|
97
|
+
depth: "moderate",
|
|
98
|
+
category: "other",
|
|
99
|
+
entities: [],
|
|
100
|
+
}, { kind: "local", surface: "cli" }));
|
|
101
|
+
}
|
|
102
|
+
export function alreadyImported(origin) {
|
|
103
|
+
const sources = loadConfig().imported_sources;
|
|
104
|
+
return Array.isArray(sources) && sources.includes(origin);
|
|
105
|
+
}
|
|
106
|
+
export function markImported(origin) {
|
|
107
|
+
const sources = loadConfig().imported_sources;
|
|
108
|
+
const list = Array.isArray(sources) ? sources : [];
|
|
109
|
+
if (!list.includes(origin))
|
|
110
|
+
saveConfig({ imported_sources: [...list, origin] });
|
|
111
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EventStore — append-only SQLite event log plus rebuildable derived views.
|
|
3
|
+
* Single source of truth per docs/EVENT_SCHEMA.md; views can always be re-derived.
|
|
4
|
+
*/
|
|
5
|
+
import { type PersnallyEvent } from "./events.js";
|
|
6
|
+
export declare const DEFAULT_DB_PATH: string;
|
|
7
|
+
export interface QueryOpts {
|
|
8
|
+
type?: string;
|
|
9
|
+
source?: string;
|
|
10
|
+
since?: string;
|
|
11
|
+
recordedSince?: string;
|
|
12
|
+
limit?: number;
|
|
13
|
+
}
|
|
14
|
+
export interface TopicRow {
|
|
15
|
+
topic_key: string;
|
|
16
|
+
topic: string;
|
|
17
|
+
category: string;
|
|
18
|
+
signals: number;
|
|
19
|
+
weight: number;
|
|
20
|
+
sentiment_balance: number;
|
|
21
|
+
dominant_intent: string;
|
|
22
|
+
entities: string[];
|
|
23
|
+
first_seen: string;
|
|
24
|
+
last_seen: string;
|
|
25
|
+
event_ids: string[];
|
|
26
|
+
}
|
|
27
|
+
export interface StoredProfile {
|
|
28
|
+
headline: string;
|
|
29
|
+
sections: {
|
|
30
|
+
title: string;
|
|
31
|
+
body: string;
|
|
32
|
+
evidence_event_ids: string[];
|
|
33
|
+
}[];
|
|
34
|
+
generated_at: string;
|
|
35
|
+
model: string;
|
|
36
|
+
}
|
|
37
|
+
export declare class EventStore {
|
|
38
|
+
private db;
|
|
39
|
+
constructor(path?: string);
|
|
40
|
+
private migrate;
|
|
41
|
+
append(events: PersnallyEvent[]): number;
|
|
42
|
+
query(opts?: QueryOpts): PersnallyEvent[];
|
|
43
|
+
getEvents(ids: string[]): PersnallyEvent[];
|
|
44
|
+
stats(): {
|
|
45
|
+
total: number;
|
|
46
|
+
byType: Record<string, number>;
|
|
47
|
+
bySource: Record<string, number>;
|
|
48
|
+
first: string | null;
|
|
49
|
+
last: string | null;
|
|
50
|
+
};
|
|
51
|
+
topics(limit?: number): TopicRow[];
|
|
52
|
+
/** Re-derive view_topics from signal.topic events using decayed per-signal weighting. */
|
|
53
|
+
rebuild(now?: number): void;
|
|
54
|
+
saveProfile(p: StoredProfile): void;
|
|
55
|
+
getProfile(): StoredProfile | null;
|
|
56
|
+
/** Hard-deletes matching topic events plus derived events referencing them, then rebuilds. */
|
|
57
|
+
forgetTopic(topic: string): number;
|
|
58
|
+
/** Removes every event from one import batch — a bad import is fully reversible. */
|
|
59
|
+
forgetBatch(batch: string): number;
|
|
60
|
+
forgetAll(): void;
|
|
61
|
+
close(): void;
|
|
62
|
+
}
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EventStore — append-only SQLite event log plus rebuildable derived views.
|
|
3
|
+
* Single source of truth per docs/EVENT_SCHEMA.md; views can always be re-derived.
|
|
4
|
+
*/
|
|
5
|
+
import Database from "better-sqlite3";
|
|
6
|
+
import { mkdirSync } from "node:fs";
|
|
7
|
+
import { dirname, join } from "node:path";
|
|
8
|
+
import { topicWeight } from "./decay.js";
|
|
9
|
+
import { normalizeTopic, validateEvent } from "./events.js";
|
|
10
|
+
import { DATA_DIR } from "./paths.js";
|
|
11
|
+
const VIEW_SCHEMA_VERSION = 2;
|
|
12
|
+
export const DEFAULT_DB_PATH = join(DATA_DIR, "persnally.db");
|
|
13
|
+
export class EventStore {
|
|
14
|
+
db;
|
|
15
|
+
constructor(path = DEFAULT_DB_PATH) {
|
|
16
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
17
|
+
this.db = new Database(path);
|
|
18
|
+
this.db.pragma("journal_mode = WAL");
|
|
19
|
+
this.migrate();
|
|
20
|
+
}
|
|
21
|
+
migrate() {
|
|
22
|
+
this.db.exec(`
|
|
23
|
+
CREATE TABLE IF NOT EXISTS events (
|
|
24
|
+
id TEXT PRIMARY KEY,
|
|
25
|
+
ts TEXT NOT NULL,
|
|
26
|
+
recorded_at TEXT NOT NULL,
|
|
27
|
+
source TEXT NOT NULL,
|
|
28
|
+
type TEXT NOT NULL,
|
|
29
|
+
payload TEXT NOT NULL,
|
|
30
|
+
provenance TEXT NOT NULL,
|
|
31
|
+
schema_ver INTEGER NOT NULL DEFAULT 1
|
|
32
|
+
);
|
|
33
|
+
CREATE INDEX IF NOT EXISTS idx_events_ts ON events (ts);
|
|
34
|
+
CREATE INDEX IF NOT EXISTS idx_events_type ON events (type, ts);
|
|
35
|
+
CREATE INDEX IF NOT EXISTS idx_events_src ON events (source, ts);
|
|
36
|
+
`);
|
|
37
|
+
// Views are derived state: on schema change, drop and re-derive rather than ALTER.
|
|
38
|
+
const ver = this.db.pragma("user_version", { simple: true }) ?? 0;
|
|
39
|
+
if (ver < VIEW_SCHEMA_VERSION) {
|
|
40
|
+
this.db.exec("DROP TABLE IF EXISTS view_topics; DROP TABLE IF EXISTS view_profile;");
|
|
41
|
+
this.db.pragma(`user_version = ${VIEW_SCHEMA_VERSION}`);
|
|
42
|
+
}
|
|
43
|
+
this.db.exec(`
|
|
44
|
+
CREATE TABLE IF NOT EXISTS view_topics (
|
|
45
|
+
topic_key TEXT PRIMARY KEY,
|
|
46
|
+
topic TEXT NOT NULL,
|
|
47
|
+
category TEXT NOT NULL,
|
|
48
|
+
signals INTEGER NOT NULL,
|
|
49
|
+
weight REAL NOT NULL,
|
|
50
|
+
sentiment_balance REAL NOT NULL,
|
|
51
|
+
dominant_intent TEXT NOT NULL,
|
|
52
|
+
entities TEXT NOT NULL,
|
|
53
|
+
first_seen TEXT NOT NULL,
|
|
54
|
+
last_seen TEXT NOT NULL,
|
|
55
|
+
event_ids TEXT NOT NULL
|
|
56
|
+
);
|
|
57
|
+
CREATE TABLE IF NOT EXISTS view_profile (
|
|
58
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
59
|
+
headline TEXT NOT NULL,
|
|
60
|
+
sections TEXT NOT NULL,
|
|
61
|
+
generated_at TEXT NOT NULL,
|
|
62
|
+
model TEXT NOT NULL
|
|
63
|
+
);
|
|
64
|
+
`);
|
|
65
|
+
// ver 0 is either a fresh db or a pre-versioning one — rebuild whenever events already exist.
|
|
66
|
+
if (ver < VIEW_SCHEMA_VERSION) {
|
|
67
|
+
const n = this.db.prepare("SELECT COUNT(*) n FROM events").get().n;
|
|
68
|
+
if (n > 0)
|
|
69
|
+
this.rebuild();
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
append(events) {
|
|
73
|
+
const insert = this.db.prepare(`INSERT INTO events (id, ts, recorded_at, source, type, payload, provenance, schema_ver)
|
|
74
|
+
VALUES (@id, @ts, @recorded_at, @source, @type, @payload, @provenance, @schema_ver)`);
|
|
75
|
+
const run = this.db.transaction((batch) => {
|
|
76
|
+
for (const raw of batch) {
|
|
77
|
+
const e = validateEvent(raw);
|
|
78
|
+
insert.run({
|
|
79
|
+
...e,
|
|
80
|
+
payload: JSON.stringify(e.payload),
|
|
81
|
+
provenance: JSON.stringify(e.provenance),
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
run(events);
|
|
86
|
+
return events.length;
|
|
87
|
+
}
|
|
88
|
+
query(opts = {}) {
|
|
89
|
+
const where = [];
|
|
90
|
+
const params = {};
|
|
91
|
+
if (opts.type) {
|
|
92
|
+
where.push("type = @type");
|
|
93
|
+
params.type = opts.type;
|
|
94
|
+
}
|
|
95
|
+
if (opts.source) {
|
|
96
|
+
where.push("source = @source");
|
|
97
|
+
params.source = opts.source;
|
|
98
|
+
}
|
|
99
|
+
if (opts.since) {
|
|
100
|
+
where.push("ts >= @since");
|
|
101
|
+
params.since = opts.since;
|
|
102
|
+
}
|
|
103
|
+
if (opts.recordedSince) {
|
|
104
|
+
where.push("recorded_at >= @recordedSince");
|
|
105
|
+
params.recordedSince = opts.recordedSince;
|
|
106
|
+
}
|
|
107
|
+
const sql = `SELECT * FROM events ${where.length ? "WHERE " + where.join(" AND ") : ""}
|
|
108
|
+
ORDER BY ts DESC LIMIT @limit`;
|
|
109
|
+
params.limit = opts.limit ?? 100;
|
|
110
|
+
return this.db.prepare(sql).all(params).map(rowToEvent);
|
|
111
|
+
}
|
|
112
|
+
getEvents(ids) {
|
|
113
|
+
if (!ids.length)
|
|
114
|
+
return [];
|
|
115
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
116
|
+
return this.db
|
|
117
|
+
.prepare(`SELECT * FROM events WHERE id IN (${placeholders})`)
|
|
118
|
+
.all(...ids)
|
|
119
|
+
.map(rowToEvent);
|
|
120
|
+
}
|
|
121
|
+
stats() {
|
|
122
|
+
const total = this.db.prepare("SELECT COUNT(*) n FROM events").get().n;
|
|
123
|
+
const group = (col) => Object.fromEntries(this.db.prepare(`SELECT ${col} k, COUNT(*) n FROM events GROUP BY ${col}`).all()
|
|
124
|
+
.map((r) => [r.k, r.n]));
|
|
125
|
+
const span = this.db.prepare("SELECT MIN(ts) first, MAX(ts) last FROM events").get();
|
|
126
|
+
return { total, byType: group("type"), bySource: group("source"), ...span };
|
|
127
|
+
}
|
|
128
|
+
topics(limit = 50) {
|
|
129
|
+
const rows = this.db
|
|
130
|
+
.prepare("SELECT * FROM view_topics ORDER BY weight DESC LIMIT ?")
|
|
131
|
+
.all(limit);
|
|
132
|
+
return rows.map((r) => ({ ...r, entities: JSON.parse(r.entities), event_ids: JSON.parse(r.event_ids) }));
|
|
133
|
+
}
|
|
134
|
+
/** Re-derive view_topics from signal.topic events using decayed per-signal weighting. */
|
|
135
|
+
rebuild(now = Date.now()) {
|
|
136
|
+
this.db.exec("DELETE FROM view_topics");
|
|
137
|
+
const acc = new Map();
|
|
138
|
+
for (const e of this.query({ type: "signal.topic", limit: 1_000_000 })) {
|
|
139
|
+
const p = e.payload;
|
|
140
|
+
const key = normalizeTopic(p.topic);
|
|
141
|
+
if (!key)
|
|
142
|
+
continue;
|
|
143
|
+
let a = acc.get(key);
|
|
144
|
+
if (!a) {
|
|
145
|
+
a = { topic: p.topic, categories: new Map(), signals: [], entities: new Set(), first: e.ts, last: e.ts, ids: [] };
|
|
146
|
+
acc.set(key, a);
|
|
147
|
+
}
|
|
148
|
+
a.categories.set(p.category, (a.categories.get(p.category) ?? 0) + 1);
|
|
149
|
+
a.signals.push({ ts: e.ts, weight: p.weight, depth: p.depth, sentiment: p.sentiment, intent: p.intent });
|
|
150
|
+
for (const ent of p.entities)
|
|
151
|
+
a.entities.add(ent);
|
|
152
|
+
if (e.ts < a.first)
|
|
153
|
+
a.first = e.ts;
|
|
154
|
+
if (e.ts > a.last)
|
|
155
|
+
a.last = e.ts;
|
|
156
|
+
a.ids.push(e.id);
|
|
157
|
+
}
|
|
158
|
+
const rows = [...acc.entries()].map(([key, a]) => {
|
|
159
|
+
const w = topicWeight(a.signals, now);
|
|
160
|
+
return {
|
|
161
|
+
topic_key: key,
|
|
162
|
+
topic: a.topic,
|
|
163
|
+
category: [...a.categories.entries()].sort((x, y) => y[1] - x[1])[0][0],
|
|
164
|
+
signals: a.signals.length,
|
|
165
|
+
weight: w.weight,
|
|
166
|
+
sentiment_balance: w.sentiment_balance,
|
|
167
|
+
dominant_intent: w.dominant_intent,
|
|
168
|
+
entities: [...a.entities].slice(0, 20),
|
|
169
|
+
first_seen: a.first,
|
|
170
|
+
last_seen: a.last,
|
|
171
|
+
event_ids: a.ids,
|
|
172
|
+
};
|
|
173
|
+
});
|
|
174
|
+
const insert = this.db.prepare(`INSERT INTO view_topics VALUES (@topic_key, @topic, @category, @signals, @weight,
|
|
175
|
+
@sentiment_balance, @dominant_intent, @entities, @first_seen, @last_seen, @event_ids)`);
|
|
176
|
+
const run = this.db.transaction((batch) => {
|
|
177
|
+
for (const r of batch) {
|
|
178
|
+
insert.run({ ...r, entities: JSON.stringify(r.entities), event_ids: JSON.stringify(r.event_ids) });
|
|
179
|
+
}
|
|
180
|
+
});
|
|
181
|
+
run(rows);
|
|
182
|
+
}
|
|
183
|
+
saveProfile(p) {
|
|
184
|
+
this.db.prepare(`INSERT INTO view_profile (id, headline, sections, generated_at, model)
|
|
185
|
+
VALUES (1, @headline, @sections, @generated_at, @model)
|
|
186
|
+
ON CONFLICT(id) DO UPDATE SET headline=@headline, sections=@sections, generated_at=@generated_at, model=@model`).run({ ...p, sections: JSON.stringify(p.sections) });
|
|
187
|
+
}
|
|
188
|
+
getProfile() {
|
|
189
|
+
const row = this.db.prepare("SELECT * FROM view_profile WHERE id = 1").get();
|
|
190
|
+
return row ? { ...row, sections: JSON.parse(row.sections) } : null;
|
|
191
|
+
}
|
|
192
|
+
/** Hard-deletes matching topic events plus derived events referencing them, then rebuilds. */
|
|
193
|
+
forgetTopic(topic) {
|
|
194
|
+
const key = normalizeTopic(topic);
|
|
195
|
+
const candidates = this.query({ type: "signal.topic", limit: 1_000_000 }).filter((e) => normalizeTopic(e.payload.topic) === key);
|
|
196
|
+
const ids = new Set(candidates.map((e) => e.id));
|
|
197
|
+
for (const e of this.query({ limit: 1_000_000 })) {
|
|
198
|
+
const prov = e.provenance;
|
|
199
|
+
if (prov.kind === "derived" && prov.from?.some((id) => ids.has(id)))
|
|
200
|
+
ids.add(e.id);
|
|
201
|
+
}
|
|
202
|
+
const del = this.db.prepare("DELETE FROM events WHERE id = ?");
|
|
203
|
+
const run = this.db.transaction((toDelete) => {
|
|
204
|
+
for (const id of toDelete)
|
|
205
|
+
del.run(id);
|
|
206
|
+
});
|
|
207
|
+
run([...ids]);
|
|
208
|
+
this.rebuild();
|
|
209
|
+
return ids.size;
|
|
210
|
+
}
|
|
211
|
+
/** Removes every event from one import batch — a bad import is fully reversible. */
|
|
212
|
+
forgetBatch(batch) {
|
|
213
|
+
const result = this.db
|
|
214
|
+
.prepare("DELETE FROM events WHERE json_extract(provenance, '$.batch') = ?")
|
|
215
|
+
.run(batch);
|
|
216
|
+
this.rebuild();
|
|
217
|
+
return result.changes;
|
|
218
|
+
}
|
|
219
|
+
forgetAll() {
|
|
220
|
+
this.db.exec("DELETE FROM events; DELETE FROM view_topics;");
|
|
221
|
+
}
|
|
222
|
+
close() {
|
|
223
|
+
this.db.close();
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
function rowToEvent(row) {
|
|
227
|
+
const r = row;
|
|
228
|
+
return {
|
|
229
|
+
...r,
|
|
230
|
+
payload: JSON.parse(r.payload),
|
|
231
|
+
provenance: JSON.parse(r.provenance),
|
|
232
|
+
};
|
|
233
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "persnally",
|
|
3
|
+
"version": "2.0.0",
|
|
4
|
+
"license": "FSL-1.1-MIT",
|
|
5
|
+
"description": "The context engine for you — local-first, across every AI. So every AI finally knows you.",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"bin": {
|
|
8
|
+
"persnally": "build/src/cli.js",
|
|
9
|
+
"persnallyd": "build/src/cli.js",
|
|
10
|
+
"persnally-mcp": "build/src/mcp/index.js"
|
|
11
|
+
},
|
|
12
|
+
"files": [
|
|
13
|
+
"build/src",
|
|
14
|
+
"LICENSE",
|
|
15
|
+
"README.md"
|
|
16
|
+
],
|
|
17
|
+
"scripts": {
|
|
18
|
+
"build": "tsc && node -e \"require('fs').copyFileSync('src/dashboard.html','build/src/dashboard.html')\"",
|
|
19
|
+
"test": "npm run build && node --test build/test/*.test.js && node test-mcp-e2e.mjs",
|
|
20
|
+
"serve": "npm run build && node build/src/cli.js serve",
|
|
21
|
+
"prepublishOnly": "node -e \"const fs=require('fs');fs.copyFileSync('../README.md','README.md');fs.copyFileSync('../LICENSE','LICENSE')\" && npm test"
|
|
22
|
+
},
|
|
23
|
+
"engines": {
|
|
24
|
+
"node": ">=20"
|
|
25
|
+
},
|
|
26
|
+
"dependencies": {
|
|
27
|
+
"@anthropic-ai/sdk": "^0.104.1",
|
|
28
|
+
"@modelcontextprotocol/sdk": "^1.5.0",
|
|
29
|
+
"better-sqlite3": "^11.8.0",
|
|
30
|
+
"zod": "^4.4.3"
|
|
31
|
+
},
|
|
32
|
+
"devDependencies": {
|
|
33
|
+
"@types/better-sqlite3": "^7.6.12",
|
|
34
|
+
"@types/node": "^22.13.5",
|
|
35
|
+
"typescript": "^5.7.3"
|
|
36
|
+
},
|
|
37
|
+
"keywords": [
|
|
38
|
+
"mcp",
|
|
39
|
+
"model-context-protocol",
|
|
40
|
+
"context-engine",
|
|
41
|
+
"personal-context",
|
|
42
|
+
"local-first",
|
|
43
|
+
"ai",
|
|
44
|
+
"claude",
|
|
45
|
+
"interest-graph"
|
|
46
|
+
],
|
|
47
|
+
"author": "Persnally",
|
|
48
|
+
"homepage": "https://persnally.com",
|
|
49
|
+
"repository": {
|
|
50
|
+
"type": "git",
|
|
51
|
+
"url": "https://github.com/sidpan2011/persnally"
|
|
52
|
+
},
|
|
53
|
+
"publishConfig": {
|
|
54
|
+
"access": "public"
|
|
55
|
+
}
|
|
56
|
+
}
|