wyrm-mcp 7.3.2 → 7.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -14
- package/dist/activation.js +59 -1
- package/dist/agent-daemon.js +281 -4
- package/dist/agent-loop.js +332 -7
- package/dist/analytics.js +236 -13
- package/dist/attribution.js +49 -1
- package/dist/audit.js +457 -2
- package/dist/auto-capture.js +138 -3
- package/dist/auto-orchestrator.js +325 -1
- package/dist/autoconfig.d.ts +50 -0
- package/dist/autoconfig.d.ts.map +1 -1
- package/dist/autoconfig.js +1115 -39
- package/dist/autoconfig.js.map +1 -1
- package/dist/buddy-runner.js +109 -1
- package/dist/buddy.js +564 -14
- package/dist/build-flags.js +15 -1
- package/dist/capabilities.js +183 -3
- package/dist/capture.js +56 -1
- package/dist/causality.js +148 -8
- package/dist/cli.js +281 -20
- package/dist/cloud/cli.js +541 -5
- package/dist/cloud/client.js +221 -1
- package/dist/cloud/crypto.js +85 -1
- package/dist/cloud/machine-id.js +113 -2
- package/dist/cloud/recovery.js +60 -1
- package/dist/cloud/sync-engine.js +543 -7
- package/dist/cloud-backup.js +579 -5
- package/dist/cloud-profile.js +138 -1
- package/dist/cloud-sync-entrypoint.js +47 -1
- package/dist/cloud-sync.js +309 -2
- package/dist/connectors/bridge-source.d.ts +46 -0
- package/dist/connectors/bridge-source.d.ts.map +1 -0
- package/dist/connectors/bridge-source.js +77 -0
- package/dist/connectors/bridge-source.js.map +1 -0
- package/dist/connectors/index.d.ts +24 -0
- package/dist/connectors/index.d.ts.map +1 -0
- package/dist/connectors/index.js +69 -0
- package/dist/connectors/index.js.map +1 -0
- package/dist/connectors/ingest.d.ts +16 -0
- package/dist/connectors/ingest.d.ts.map +1 -0
- package/dist/connectors/ingest.js +116 -0
- package/dist/connectors/ingest.js.map +1 -0
- package/dist/connectors/types.d.ts +99 -0
- package/dist/connectors/types.d.ts.map +1 -0
- package/dist/connectors/types.js +17 -0
- package/dist/connectors/types.js.map +1 -0
- package/dist/constellation.js +168 -12
- package/dist/content-signature.js +45 -1
- package/dist/context-build-budgeted.js +144 -4
- package/dist/context-ranking.js +69 -1
- package/dist/crypto.js +179 -1
- package/dist/daemon-write-endpoint.js +290 -1
- package/dist/daemon-writer.js +406 -2
- package/dist/database.js +1278 -53
- package/dist/deprecations.js +162 -2
- package/dist/design.js +141 -13
- package/dist/event-replication.js +112 -1
- package/dist/events-sse.js +43 -7
- package/dist/events.js +238 -6
- package/dist/failure-patterns.d.ts +107 -0
- package/dist/failure-patterns.d.ts.map +1 -1
- package/dist/failure-patterns.js +924 -43
- package/dist/failure-patterns.js.map +1 -1
- package/dist/federation.js +236 -12
- package/dist/goals.js +101 -13
- package/dist/golden.js +355 -3
- package/dist/handlers/agent.js +165 -4
- package/dist/handlers/alias-adapters.js +129 -1
- package/dist/handlers/aliases.js +171 -1
- package/dist/handlers/audit.js +87 -1
- package/dist/handlers/boundary.js +221 -1
- package/dist/handlers/capture.js +1114 -73
- package/dist/handlers/causality.js +119 -9
- package/dist/handlers/cloud.js +382 -85
- package/dist/handlers/companion.js +459 -28
- package/dist/handlers/datalake.js +187 -7
- package/dist/handlers/dispatch-context.js +22 -0
- package/dist/handlers/entity.js +256 -25
- package/dist/handlers/events.js +335 -16
- package/dist/handlers/failure.d.ts.map +1 -1
- package/dist/handlers/failure.js +408 -13
- package/dist/handlers/failure.js.map +1 -1
- package/dist/handlers/goals.js +296 -4
- package/dist/handlers/intelligence.js +681 -126
- package/dist/handlers/invoicing.js +70 -1
- package/dist/handlers/mcpclient.js +137 -6
- package/dist/handlers/orchestration.js +125 -40
- package/dist/handlers/output-schemas.js +24 -1
- package/dist/handlers/presence.js +99 -3
- package/dist/handlers/project.js +182 -28
- package/dist/handlers/prompts.js +157 -6
- package/dist/handlers/quest.js +224 -4
- package/dist/handlers/recall.js +237 -13
- package/dist/handlers/registry.js +167 -1
- package/dist/handlers/resources.js +288 -1
- package/dist/handlers/review.js +74 -11
- package/dist/handlers/run.js +498 -16
- package/dist/handlers/search.js +338 -15
- package/dist/handlers/session.js +643 -31
- package/dist/handlers/share.js +184 -8
- package/dist/handlers/shims.js +464 -1
- package/dist/handlers/skill.js +449 -67
- package/dist/handlers/survivors.js +120 -1
- package/dist/handlers/symbols.js +109 -8
- package/dist/handlers/syncops.js +302 -4
- package/dist/handlers/types.js +27 -1
- package/dist/harvest.js +191 -5
- package/dist/hours.js +156 -7
- package/dist/http-auth.js +321 -3
- package/dist/http-fast.js +1302 -22
- package/dist/icons.js +47 -1
- package/dist/importers.js +268 -1
- package/dist/index.js +840 -2
- package/dist/indexer.js +145 -4
- package/dist/intelligence.js +261 -31
- package/dist/internal-dispatch.js +212 -3
- package/dist/keyset.js +110 -1
- package/dist/knowledge-graph.js +176 -12
- package/dist/license.js +441 -2
- package/dist/logger.js +199 -2
- package/dist/maintenance.js +148 -2
- package/dist/mcp-client.js +262 -6
- package/dist/memory-artifacts.js +596 -32
- package/dist/migrate-prompt.js +124 -2
- package/dist/migrations.d.ts.map +1 -1
- package/dist/migrations.js +799 -42
- package/dist/migrations.js.map +1 -1
- package/dist/performance.js +228 -1
- package/dist/presence.js +140 -11
- package/dist/priority-embed.js +164 -5
- package/dist/providers/embedding-provider.js +196 -1
- package/dist/readonly-gate.js +29 -1
- package/dist/receipt.js +43 -1
- package/dist/rehydration.js +157 -9
- package/dist/reindex.js +88 -1
- package/dist/render-target.js +544 -21
- package/dist/render.js +280 -4
- package/dist/repl-guard.js +173 -1
- package/dist/replication-daemon-entrypoint.js +31 -1
- package/dist/replication-daemon.js +262 -2
- package/dist/rerank.js +142 -1
- package/dist/resilience.js +591 -1
- package/dist/reverse-bridge.js +360 -5
- package/dist/security.js +244 -1
- package/dist/session-seen.js +51 -3
- package/dist/setup.js +260 -1
- package/dist/skill-author.js +168 -5
- package/dist/spec-kit.js +191 -1
- package/dist/sqlite-busy.js +154 -1
- package/dist/statusline.js +315 -11
- package/dist/sub-agent.js +262 -13
- package/dist/summarizer.js +139 -13
- package/dist/symbols.js +283 -7
- package/dist/sync.js +359 -5
- package/dist/tasks-dispatch.js +84 -1
- package/dist/tasks.js +282 -1
- package/dist/token-budget.js +143 -1
- package/dist/tool-analytics.js +129 -7
- package/dist/tool-annotations.js +365 -1
- package/dist/tool-manifest-v2.json +1 -1
- package/dist/tool-manifest.json +1 -1
- package/dist/tool-profiles.js +75 -1
- package/dist/trace-harvest.js +244 -6
- package/dist/types.js +30 -1
- package/dist/ui-dashboard.js +50 -41
- package/dist/ulid.js +81 -1
- package/dist/usage-tracker.js +66 -1
- package/dist/validate.js +129 -1
- package/dist/vault.js +534 -1
- package/dist/vector-init.js +67 -1
- package/dist/vectors.js +184 -3
- package/dist/version-check.js +136 -4
- package/dist/visibility.js +155 -19
- package/dist/wyrm-cli.js +2845 -101
- package/dist/wyrm-cli.js.map +1 -1
- package/dist/wyrm-guard.d.ts.map +1 -1
- package/dist/wyrm-guard.js +475 -14
- package/dist/wyrm-guard.js.map +1 -1
- package/dist/wyrm-loop.js +150 -3
- package/dist/wyrm-manifest.json +1 -1
- package/dist/wyrm-statusline-daemon.js +11 -1
- package/dist/wyrm-statusline.js +56 -4
- package/dist/wyrm-ui.js +77 -9
- package/package.json +1 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/connectors/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAG7C,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE3F,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAI3C,0DAA0D;AAC1D,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,MAAM,GAAG,eAAe,EAAE,CASlE;AAED,yDAAyD;AACzD,wBAAgB,oBAAoB,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,IAAI,CAEjF;AAED,0CAA0C;AAC1C,wBAAgB,qBAAqB,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,GAAG,eAAe,EAAE,CAK5F;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,eAAe,CAarE;AAED,wCAAwC;AACxC,wBAAsB,YAAY,CAChC,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,eAAe,EACvB,IAAI,GAAE,aAAkB,GACvB,OAAO,CAAC,YAAY,CAAC,CAGvB;AAED,2EAA2E;AAC3E,wBAAsB,gBAAgB,CACpC,EAAE,EAAE,MAAM,EACV,IAAI,GAAE,aAAkB,GACvB,OAAO,CAAC,KAAK,CAAC,YAAY,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC,CAWhE"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { BridgeSource } from './bridge-source.js';
|
|
2
|
+
import { runIngestion } from './ingest.js';
|
|
3
|
+
export * from './types.js';
|
|
4
|
+
export { BridgeSource } from './bridge-source.js';
|
|
5
|
+
export { runIngestion } from './ingest.js';
|
|
6
|
+
const CONFIG_KEY = 'connector_config';
|
|
7
|
+
/** Load all configured connectors from global_context. */
|
|
8
|
+
export function loadConnectorConfigs(db) {
|
|
9
|
+
const raw = db.getGlobalContext(CONFIG_KEY);
|
|
10
|
+
if (!raw)
|
|
11
|
+
return [];
|
|
12
|
+
try {
|
|
13
|
+
const parsed = JSON.parse(raw);
|
|
14
|
+
return Array.isArray(parsed) ? parsed : [];
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return [];
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/** Persist the full connector list to global_context. */
|
|
21
|
+
export function saveConnectorConfigs(db, configs) {
|
|
22
|
+
db.setGlobalContext(CONFIG_KEY, JSON.stringify(configs));
|
|
23
|
+
}
|
|
24
|
+
/** Add or replace a connector by name. */
|
|
25
|
+
export function upsertConnectorConfig(db, config) {
|
|
26
|
+
const configs = loadConnectorConfigs(db).filter((c) => c.name !== config.name);
|
|
27
|
+
configs.push(config);
|
|
28
|
+
saveConnectorConfigs(db, configs);
|
|
29
|
+
return configs;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Build a live source from a config. The auth token is read from the
|
|
33
|
+
* WYRM_CONNECTOR_TOKEN_<NAME> env var if present, falling back to config.authToken.
|
|
34
|
+
*/
|
|
35
|
+
export function createSource(config) {
|
|
36
|
+
const envToken = process.env[`WYRM_CONNECTOR_TOKEN_${config.name.toUpperCase().replace(/[^A-Z0-9]/g, '_')}`];
|
|
37
|
+
switch (config.source) {
|
|
38
|
+
case 'bridge':
|
|
39
|
+
return new BridgeSource({
|
|
40
|
+
name: config.name,
|
|
41
|
+
baseUrl: config.baseUrl,
|
|
42
|
+
authToken: envToken ?? config.authToken,
|
|
43
|
+
allowlist: config.allowlist,
|
|
44
|
+
});
|
|
45
|
+
default:
|
|
46
|
+
throw new Error(`Unknown connector source: ${String(config.source)}`);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
/** Run a single connector by config. */
|
|
50
|
+
export async function runConnector(db, config, opts = {}) {
|
|
51
|
+
const source = createSource(config);
|
|
52
|
+
return runIngestion(db, config, source, opts);
|
|
53
|
+
}
|
|
54
|
+
/** Run every enabled connector once. Errors are isolated per connector. */
|
|
55
|
+
export async function runAllConnectors(db, opts = {}) {
|
|
56
|
+
const out = [];
|
|
57
|
+
for (const config of loadConnectorConfigs(db)) {
|
|
58
|
+
if (!config.enabled)
|
|
59
|
+
continue;
|
|
60
|
+
try {
|
|
61
|
+
out.push(await runConnector(db, config, opts));
|
|
62
|
+
}
|
|
63
|
+
catch (err) {
|
|
64
|
+
out.push({ name: config.name, error: String(err) });
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return out;
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/connectors/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,MAAM,UAAU,GAAG,kBAAkB,CAAC;AAEtC,0DAA0D;AAC1D,MAAM,UAAU,oBAAoB,CAAC,EAAU;IAC7C,MAAM,GAAG,GAAG,EAAE,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAC5C,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAY,CAAC;QAC1C,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAE,MAA4B,CAAC,CAAC,CAAC,EAAE,CAAC;IACpE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,oBAAoB,CAAC,EAAU,EAAE,OAA0B;IACzE,EAAE,CAAC,gBAAgB,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,0CAA0C;AAC1C,MAAM,UAAU,qBAAqB,CAAC,EAAU,EAAE,MAAuB;IACvE,MAAM,OAAO,GAAG,oBAAoB,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC;IAC/E,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrB,oBAAoB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IAClC,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,MAAuB;IAClD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC7G,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;QACtB,KAAK,QAAQ;YACX,OAAO,IAAI,YAAY,CAAC;gBACtB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,SAAS,EAAE,QAAQ,IAAI,MAAM,CAAC,SAAS;gBACvC,SAAS,EAAE,MAAM,CAAC,SAAS;aAC5B,CAAC,CAAC;QACL;YACE,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAE,MAA0B,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC/F,CAAC;AACH,CAAC;AAED,wCAAwC;AACxC,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,EAAU,EACV,MAAuB,EACvB,OAAsB,EAAE;IAExB,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACpC,OAAO,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;AAChD,CAAC;AAED,2EAA2E;AAC3E,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,EAAU,EACV,OAAsB,EAAE;IAExB,MAAM,GAAG,GAA0D,EAAE,CAAC;IACtE,KAAK,MAAM,MAAM,IAAI,oBAAoB,CAAC,EAAE,CAAC,EAAE,CAAC;QAC9C,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,SAAS;QAC9B,IAAI,CAAC;YACH,GAAG,CAAC,IAAI,CAAC,MAAM,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;QACjD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { WyrmDB } from '../database.js';
|
|
2
|
+
import { ConnectorConfig, IngestionSource, IngestOptions, IngestResult } from './types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Run one ingestion pass for a connector:
|
|
5
|
+
* 1. fetch new messages from the source (since the stored cursor),
|
|
6
|
+
* 2. drop allowlist/duplicate messages,
|
|
7
|
+
* 3. write a raw transcript snapshot to the data_lake (context never lost),
|
|
8
|
+
* 4. optionally run the LLM `distill` seam to write truths/quests,
|
|
9
|
+
* 5. advance the cursor + seen-set.
|
|
10
|
+
*
|
|
11
|
+
* Cloud sync is intentionally NOT triggered here: the existing cloud-sync daemon
|
|
12
|
+
* picks up the new rows as deltas. Keeping them decoupled means a sync outage
|
|
13
|
+
* never blocks ingestion (and vice versa).
|
|
14
|
+
*/
|
|
15
|
+
export declare function runIngestion(db: WyrmDB, config: ConnectorConfig, source: IngestionSource, opts?: IngestOptions): Promise<IngestResult>;
|
|
16
|
+
//# sourceMappingURL=ingest.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../src/connectors/ingest.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAE7C,OAAO,EACL,eAAe,EACf,eAAe,EACf,aAAa,EACb,YAAY,EAEb,MAAM,YAAY,CAAC;AA4BpB;;;;;;;;;;;GAWG;AACH,wBAAsB,YAAY,CAChC,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,eAAe,EACvB,MAAM,EAAE,eAAe,EACvB,IAAI,GAAE,aAAkB,GACvB,OAAO,CAAC,YAAY,CAAC,CAwDvB"}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { classifyCapture } from '../capture.js';
|
|
2
|
+
const DEFAULT_RAW_CATEGORY = 'ingested_transcripts';
|
|
3
|
+
const DEFAULT_SEEN_CAP = 5000;
|
|
4
|
+
function cursorKey(name) {
|
|
5
|
+
return `connector:${name}:cursor`;
|
|
6
|
+
}
|
|
7
|
+
function seenKey(name) {
|
|
8
|
+
return `connector:${name}:seen`;
|
|
9
|
+
}
|
|
10
|
+
/** Resolve (or lazily create) the target workspace/project and return its id. */
|
|
11
|
+
function resolveProjectId(db, workspace) {
|
|
12
|
+
const existing = db.getProjectByName(workspace);
|
|
13
|
+
if (existing)
|
|
14
|
+
return existing.id;
|
|
15
|
+
// Logical workspace with no filesystem path: register under a connector:// path.
|
|
16
|
+
const created = db.registerProject(workspace, `connector://${workspace}`);
|
|
17
|
+
return created.id;
|
|
18
|
+
}
|
|
19
|
+
/** Render new messages into a readable transcript snapshot. */
|
|
20
|
+
function renderTranscript(messages) {
|
|
21
|
+
return messages
|
|
22
|
+
.map((m) => `[${m.ts}] ${m.sender} (${m.chat}): ${m.text}`)
|
|
23
|
+
.join('\n');
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Run one ingestion pass for a connector:
|
|
27
|
+
* 1. fetch new messages from the source (since the stored cursor),
|
|
28
|
+
* 2. drop allowlist/duplicate messages,
|
|
29
|
+
* 3. write a raw transcript snapshot to the data_lake (context never lost),
|
|
30
|
+
* 4. optionally run the LLM `distill` seam to write truths/quests,
|
|
31
|
+
* 5. advance the cursor + seen-set.
|
|
32
|
+
*
|
|
33
|
+
* Cloud sync is intentionally NOT triggered here: the existing cloud-sync daemon
|
|
34
|
+
* picks up the new rows as deltas. Keeping them decoupled means a sync outage
|
|
35
|
+
* never blocks ingestion (and vice versa).
|
|
36
|
+
*/
|
|
37
|
+
export async function runIngestion(db, config, source, opts = {}) {
|
|
38
|
+
const now = opts.now ?? (() => new Date());
|
|
39
|
+
const seenCap = opts.seenCap ?? DEFAULT_SEEN_CAP;
|
|
40
|
+
const rawCategory = config.rawCategory ?? DEFAULT_RAW_CATEGORY;
|
|
41
|
+
const projectId = resolveProjectId(db, config.workspace);
|
|
42
|
+
const prevCursor = db.getGlobalContext(cursorKey(source.name)) ?? null;
|
|
43
|
+
const seen = loadSeen(db, source.name);
|
|
44
|
+
const { messages, cursor } = await source.fetchSince(prevCursor);
|
|
45
|
+
// De-duplicate against the seen-set (survives restarts; cursor alone can overlap).
|
|
46
|
+
const fresh = [];
|
|
47
|
+
let skipped = 0;
|
|
48
|
+
for (const m of messages) {
|
|
49
|
+
if (seen.has(m.id)) {
|
|
50
|
+
skipped++;
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
fresh.push(m);
|
|
54
|
+
seen.add(m.id);
|
|
55
|
+
}
|
|
56
|
+
const result = {
|
|
57
|
+
source: source.name,
|
|
58
|
+
workspace: config.workspace,
|
|
59
|
+
fetched: messages.length,
|
|
60
|
+
ingested: fresh.length,
|
|
61
|
+
skipped,
|
|
62
|
+
cursor,
|
|
63
|
+
};
|
|
64
|
+
if (fresh.length > 0) {
|
|
65
|
+
const stamp = now().toISOString().replace(/[:.]/g, '-');
|
|
66
|
+
const rawKey = `${source.name}_${stamp}`;
|
|
67
|
+
const typeCounts = tallyTypes(fresh);
|
|
68
|
+
db.insertData(projectId, rawCategory, rawKey, renderTranscript(fresh), {
|
|
69
|
+
source: source.name,
|
|
70
|
+
count: fresh.length,
|
|
71
|
+
chats: [...new Set(fresh.map((m) => m.chat))],
|
|
72
|
+
typeCounts,
|
|
73
|
+
cursor,
|
|
74
|
+
});
|
|
75
|
+
result.rawKey = rawKey;
|
|
76
|
+
if (opts.distill) {
|
|
77
|
+
await opts.distill(fresh, { workspace: config.workspace, projectId });
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
// Persist cursor + capped seen-set for the next run.
|
|
81
|
+
db.setGlobalContext(cursorKey(source.name), cursor);
|
|
82
|
+
saveSeen(db, source.name, seen, seenCap);
|
|
83
|
+
return result;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Cheap, no-LLM signal: tally how each new message would classify (quest / truth
|
|
87
|
+
* / memory). Stored in the snapshot metadata so a later distiller — or a human —
|
|
88
|
+
* can see at a glance what kind of content arrived.
|
|
89
|
+
*/
|
|
90
|
+
function tallyTypes(messages) {
|
|
91
|
+
const counts = {};
|
|
92
|
+
for (const m of messages) {
|
|
93
|
+
const { type } = classifyCapture(m.text);
|
|
94
|
+
counts[type] = (counts[type] ?? 0) + 1;
|
|
95
|
+
}
|
|
96
|
+
return counts;
|
|
97
|
+
}
|
|
98
|
+
function loadSeen(db, name) {
|
|
99
|
+
const raw = db.getGlobalContext(seenKey(name));
|
|
100
|
+
if (!raw)
|
|
101
|
+
return new Set();
|
|
102
|
+
try {
|
|
103
|
+
const arr = JSON.parse(raw);
|
|
104
|
+
return new Set(Array.isArray(arr) ? arr.map(String) : []);
|
|
105
|
+
}
|
|
106
|
+
catch {
|
|
107
|
+
return new Set();
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
function saveSeen(db, name, seen, cap) {
|
|
111
|
+
// Keep only the most-recent `cap` ids (insertion order is preserved by Set).
|
|
112
|
+
const arr = [...seen];
|
|
113
|
+
const trimmed = arr.length > cap ? arr.slice(arr.length - cap) : arr;
|
|
114
|
+
db.setGlobalContext(seenKey(name), JSON.stringify(trimmed));
|
|
115
|
+
}
|
|
116
|
+
//# sourceMappingURL=ingest.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../src/connectors/ingest.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAShD,MAAM,oBAAoB,GAAG,sBAAsB,CAAC;AACpD,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,aAAa,IAAI,SAAS,CAAC;AACpC,CAAC;AACD,SAAS,OAAO,CAAC,IAAY;IAC3B,OAAO,aAAa,IAAI,OAAO,CAAC;AAClC,CAAC;AAED,iFAAiF;AACjF,SAAS,gBAAgB,CAAC,EAAU,EAAE,SAAiB;IACrD,MAAM,QAAQ,GAAG,EAAE,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IAChD,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC,EAAE,CAAC;IACjC,iFAAiF;IACjF,MAAM,OAAO,GAAG,EAAE,CAAC,eAAe,CAAC,SAAS,EAAE,eAAe,SAAS,EAAE,CAAC,CAAC;IAC1E,OAAO,OAAO,CAAC,EAAE,CAAC;AACpB,CAAC;AAED,+DAA+D;AAC/D,SAAS,gBAAgB,CAAC,QAAsB;IAC9C,OAAO,QAAQ;SACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;SAC1D,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,EAAU,EACV,MAAuB,EACvB,MAAuB,EACvB,OAAsB,EAAE;IAExB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,gBAAgB,CAAC;IACjD,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,oBAAoB,CAAC;IAE/D,MAAM,SAAS,GAAG,gBAAgB,CAAC,EAAE,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IAEzD,MAAM,UAAU,GAAG,EAAE,CAAC,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC;IACvE,MAAM,IAAI,GAAG,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;IAEvC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;IAEjE,mFAAmF;IACnF,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;YACnB,OAAO,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACd,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,MAAM,GAAiB;QAC3B,MAAM,EAAE,MAAM,CAAC,IAAI;QACnB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,OAAO,EAAE,QAAQ,CAAC,MAAM;QACxB,QAAQ,EAAE,KAAK,CAAC,MAAM;QACtB,OAAO;QACP,MAAM;KACP,CAAC;IAEF,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,IAAI,KAAK,EAAE,CAAC;QACzC,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QACrC,EAAE,CAAC,UAAU,CAAC,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,gBAAgB,CAAC,KAAK,CAAC,EAAE;YACrE,MAAM,EAAE,MAAM,CAAC,IAAI;YACnB,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,KAAK,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YAC7C,UAAU;YACV,MAAM;SACP,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;QAEvB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IAED,qDAAqD;IACrD,EAAE,CAAC,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;IACpD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IAEzC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,QAAsB;IACxC,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,EAAE,IAAI,EAAE,GAAG,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,EAAU,EAAE,IAAY;IACxC,MAAM,GAAG,GAAG,EAAE,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/C,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,GAAG,EAAE,CAAC;IAC3B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAY,CAAC;QACvC,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC5D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,GAAG,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,EAAU,EAAE,IAAY,EAAE,IAAiB,EAAE,GAAW;IACxE,6EAA6E;IAC7E,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACtB,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IACrE,EAAE,CAAC,gBAAgB,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;AAC9D,CAAC"}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ingestion connectors — pull messages from an external, always-on source
|
|
3
|
+
* (e.g. a tunnelled WhatsApp bridge) into Wyrm memory.
|
|
4
|
+
*
|
|
5
|
+
* Design note: the source itself (the thing holding the live WhatsApp/Slack
|
|
6
|
+
* session) lives OUTSIDE Wyrm — it is a "dumb bridge" reachable over a private,
|
|
7
|
+
* authenticated tunnel. This module is the Wyrm-side puller/ingester: it fetches
|
|
8
|
+
* new messages since a stored cursor, writes a raw transcript snapshot to the
|
|
9
|
+
* data_lake (so context is never lost), and exposes a `distill` seam for an LLM
|
|
10
|
+
* pass that turns the transcript into truths/quests. Nothing here holds a chat
|
|
11
|
+
* session, so it cannot crash a chat agent, and it runs client-side before any
|
|
12
|
+
* cloud encryption (so it respects Wyrm Cloud's zero-knowledge model).
|
|
13
|
+
*
|
|
14
|
+
* See docs/ingestion-connectors.md for the full architecture.
|
|
15
|
+
*/
|
|
16
|
+
/** A single normalised inbound message from any source. */
|
|
17
|
+
export interface RawMessage {
|
|
18
|
+
/** Stable unique id from the source, used for de-duplication. */
|
|
19
|
+
id: string;
|
|
20
|
+
/** ISO-8601 timestamp of the message. */
|
|
21
|
+
ts: string;
|
|
22
|
+
/** Display name or handle of the sender. */
|
|
23
|
+
sender: string;
|
|
24
|
+
/** Message body (plain text). */
|
|
25
|
+
text: string;
|
|
26
|
+
/** Chat / group name or id the message belongs to. */
|
|
27
|
+
chat: string;
|
|
28
|
+
}
|
|
29
|
+
/** Result of a single fetch from a source. */
|
|
30
|
+
export interface FetchResult {
|
|
31
|
+
messages: RawMessage[];
|
|
32
|
+
/** Opaque cursor to pass back on the next fetch to get only newer messages. */
|
|
33
|
+
cursor: string;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* A pluggable message source. Implementations: BridgeSource (WhatsApp bridge).
|
|
37
|
+
* Future: SlackSource, GmailSource, FirefliesSource — same contract.
|
|
38
|
+
*/
|
|
39
|
+
export interface IngestionSource {
|
|
40
|
+
/** Stable identifier for this source instance (used in cursor/seen keys). */
|
|
41
|
+
readonly name: string;
|
|
42
|
+
/** Fetch messages newer than `cursor` (null = from the beginning the source allows). */
|
|
43
|
+
fetchSince(cursor: string | null): Promise<FetchResult>;
|
|
44
|
+
}
|
|
45
|
+
/** Persisted configuration for one connector. Stored in global_context. */
|
|
46
|
+
export interface ConnectorConfig {
|
|
47
|
+
/** Unique name for this connector (e.g. "j2i-whatsapp"). */
|
|
48
|
+
name: string;
|
|
49
|
+
/** Source type. Currently only "bridge". */
|
|
50
|
+
source: 'bridge';
|
|
51
|
+
/** Whether `runAllConnectors` includes this one. */
|
|
52
|
+
enabled: boolean;
|
|
53
|
+
/** Base URL of the tunnelled bridge endpoint (e.g. https://bridge.example.ts.net). */
|
|
54
|
+
baseUrl: string;
|
|
55
|
+
/**
|
|
56
|
+
* Bearer token for the bridge. Prefer the WYRM_CONNECTOR_TOKEN_<NAME> env var
|
|
57
|
+
* over storing it here; tokens in config are never logged.
|
|
58
|
+
*/
|
|
59
|
+
authToken?: string;
|
|
60
|
+
/** Chat ids/names to ingest. Empty array = ingest everything the source returns. */
|
|
61
|
+
allowlist: string[];
|
|
62
|
+
/** Target Wyrm project/workspace name to file ingested data + memories under. */
|
|
63
|
+
workspace: string;
|
|
64
|
+
/** data_lake category for the raw transcript snapshot. Default "ingested_transcripts". */
|
|
65
|
+
rawCategory?: string;
|
|
66
|
+
}
|
|
67
|
+
/** Outcome of one ingestion run. */
|
|
68
|
+
export interface IngestResult {
|
|
69
|
+
source: string;
|
|
70
|
+
workspace: string;
|
|
71
|
+
/** Messages returned by the source this run. */
|
|
72
|
+
fetched: number;
|
|
73
|
+
/** New (de-duplicated, allowlisted) messages stored this run. */
|
|
74
|
+
ingested: number;
|
|
75
|
+
/** Messages dropped by the allowlist or already seen. */
|
|
76
|
+
skipped: number;
|
|
77
|
+
/** New cursor persisted for the next run. */
|
|
78
|
+
cursor: string;
|
|
79
|
+
/** data_lake key of the raw snapshot written this run, if any. */
|
|
80
|
+
rawKey?: string;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Optional LLM distillation seam. Given the new messages, the implementer writes
|
|
84
|
+
* truths/quests into the workspace (via Claude API, local Ollama, or Wyrm's own
|
|
85
|
+
* extractor). Left as a seam because the model choice is deployment-specific.
|
|
86
|
+
*/
|
|
87
|
+
export type Distiller = (messages: RawMessage[], ctx: {
|
|
88
|
+
workspace: string;
|
|
89
|
+
projectId: number;
|
|
90
|
+
}) => Promise<void> | void;
|
|
91
|
+
export interface IngestOptions {
|
|
92
|
+
/** Optional LLM distillation pass run after the raw snapshot is stored. */
|
|
93
|
+
distill?: Distiller;
|
|
94
|
+
/** Injectable clock for deterministic tests. */
|
|
95
|
+
now?: () => Date;
|
|
96
|
+
/** Cap on the de-dup "seen" set kept per connector. Default 5000. */
|
|
97
|
+
seenCap?: number;
|
|
98
|
+
}
|
|
99
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/connectors/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,2DAA2D;AAC3D,MAAM,WAAW,UAAU;IACzB,iEAAiE;IACjE,EAAE,EAAE,MAAM,CAAC;IACX,yCAAyC;IACzC,EAAE,EAAE,MAAM,CAAC;IACX,4CAA4C;IAC5C,MAAM,EAAE,MAAM,CAAC;IACf,iCAAiC;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,sDAAsD;IACtD,IAAI,EAAE,MAAM,CAAC;CACd;AAED,8CAA8C;AAC9C,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,UAAU,EAAE,CAAC;IACvB,+EAA+E;IAC/E,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,6EAA6E;IAC7E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,wFAAwF;IACxF,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;CACzD;AAED,2EAA2E;AAC3E,MAAM,WAAW,eAAe;IAC9B,4DAA4D;IAC5D,IAAI,EAAE,MAAM,CAAC;IACb,4CAA4C;IAC5C,MAAM,EAAE,QAAQ,CAAC;IACjB,oDAAoD;IACpD,OAAO,EAAE,OAAO,CAAC;IACjB,sFAAsF;IACtF,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oFAAoF;IACpF,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,iFAAiF;IACjF,SAAS,EAAE,MAAM,CAAC;IAClB,0FAA0F;IAC1F,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,oCAAoC;AACpC,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,gDAAgD;IAChD,OAAO,EAAE,MAAM,CAAC;IAChB,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAC;IACjB,yDAAyD;IACzD,OAAO,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,kEAAkE;IAClE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;GAIG;AACH,MAAM,MAAM,SAAS,GAAG,CACtB,QAAQ,EAAE,UAAU,EAAE,EACtB,GAAG,EAAE;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,KAC1C,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAE1B,MAAM,WAAW,aAAa;IAC5B,2EAA2E;IAC3E,OAAO,CAAC,EAAE,SAAS,CAAC;IACpB,gDAAgD;IAChD,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IACjB,qEAAqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ingestion connectors — pull messages from an external, always-on source
|
|
3
|
+
* (e.g. a tunnelled WhatsApp bridge) into Wyrm memory.
|
|
4
|
+
*
|
|
5
|
+
* Design note: the source itself (the thing holding the live WhatsApp/Slack
|
|
6
|
+
* session) lives OUTSIDE Wyrm — it is a "dumb bridge" reachable over a private,
|
|
7
|
+
* authenticated tunnel. This module is the Wyrm-side puller/ingester: it fetches
|
|
8
|
+
* new messages since a stored cursor, writes a raw transcript snapshot to the
|
|
9
|
+
* data_lake (so context is never lost), and exposes a `distill` seam for an LLM
|
|
10
|
+
* pass that turns the transcript into truths/quests. Nothing here holds a chat
|
|
11
|
+
* session, so it cannot crash a chat agent, and it runs client-side before any
|
|
12
|
+
* cloud encryption (so it respects Wyrm Cloud's zero-knowledge model).
|
|
13
|
+
*
|
|
14
|
+
* See docs/ingestion-connectors.md for the full architecture.
|
|
15
|
+
*/
|
|
16
|
+
export {};
|
|
17
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/connectors/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG"}
|
package/dist/constellation.js
CHANGED
|
@@ -1,37 +1,193 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* wyrm_constellation — cross-project memory query (spec 018 phase 3).
|
|
3
|
+
*
|
|
4
|
+
* Asks one FTS5 question across *every* registered Wyrm project — not just
|
|
5
|
+
* the current one. The killer founder-grade query: "have I solved this
|
|
6
|
+
* before in any project?" Returns candidates grouped by project; the
|
|
7
|
+
* calling AI does semantic ranking on its side (per spec 018 decision #2 —
|
|
8
|
+
* no embedding dependency).
|
|
9
|
+
*
|
|
10
|
+
* Privacy: respects the `cross_project_visibility` flag (added by
|
|
11
|
+
* migration 14, default `'within'`). Only artifacts explicitly flagged
|
|
12
|
+
* `'org'` or `'public'` leak across the project boundary.
|
|
13
|
+
*
|
|
14
|
+
* @copyright 2026 Ghost Protocol (Pvt) Ltd.
|
|
15
|
+
* @license Proprietary — (c) 2026 Ghost Protocol (Pvt) Ltd. All rights reserved. See LICENSE.
|
|
16
|
+
*/
|
|
17
|
+
const SAFE_KINDS = new Set(['truth', 'artifact', 'quest', 'decision', 'reference']);
|
|
18
|
+
const SAFE_VIS = new Set(['within', 'org', 'public']);
|
|
19
|
+
function sanitizeFtsQuery(q) {
|
|
20
|
+
// FTS5 syntax allows `"`, `*`, `AND`, `OR`, `NOT`. We strip anything that
|
|
21
|
+
// could close out into raw SQL — only allow word chars, spaces, quotes,
|
|
22
|
+
// asterisk, +/-. This is defense-in-depth; we already use parameterized
|
|
23
|
+
// bind, but the query also drives string-concat in some paths.
|
|
24
|
+
return q.replace(/[^\w\s"\*\-\+]/g, ' ').trim().slice(0, 200);
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Query a single project's FTS5 indexes for matching artifacts.
|
|
28
|
+
* Pre-filtered to honour visibility; returns up to `limit` hits.
|
|
29
|
+
*/
|
|
30
|
+
function queryProject(db, project, query, kinds, visibility, limit) {
|
|
31
|
+
const hits = [];
|
|
32
|
+
const visList = `(${[...visibility].map((v) => `'${v}'`).join(',')})`;
|
|
33
|
+
const safeQ = sanitizeFtsQuery(query);
|
|
34
|
+
if (!safeQ)
|
|
35
|
+
return [];
|
|
36
|
+
// truths
|
|
37
|
+
if (kinds.has('truth')) {
|
|
38
|
+
try {
|
|
39
|
+
const rows = db.prepare(`
|
|
2
40
|
SELECT id, key, value, cross_project_visibility AS visibility, created_at
|
|
3
41
|
FROM ground_truths
|
|
4
42
|
WHERE project_id = ?
|
|
5
|
-
AND cross_project_visibility IN ${
|
|
43
|
+
AND cross_project_visibility IN ${visList}
|
|
6
44
|
AND (key LIKE '%' || ? || '%' OR value LIKE '%' || ? || '%')
|
|
7
45
|
ORDER BY created_at DESC
|
|
8
46
|
LIMIT ?
|
|
9
|
-
`).all(
|
|
47
|
+
`).all(project.id, safeQ, safeQ, limit);
|
|
48
|
+
for (const r of rows) {
|
|
49
|
+
hits.push({
|
|
50
|
+
projectId: project.id,
|
|
51
|
+
projectName: project.name,
|
|
52
|
+
kind: 'truth',
|
|
53
|
+
id: r.id,
|
|
54
|
+
title: r.key,
|
|
55
|
+
snippet: r.value.slice(0, 200),
|
|
56
|
+
visibility: r.visibility,
|
|
57
|
+
capturedAt: r.created_at,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
catch { /* table or column may not exist on older schema */ }
|
|
62
|
+
}
|
|
63
|
+
// memory artifacts
|
|
64
|
+
if (kinds.has('artifact')) {
|
|
65
|
+
try {
|
|
66
|
+
const rows = db.prepare(`
|
|
10
67
|
SELECT ma.id, ma.problem AS title, ma.validated_fix AS snippet,
|
|
11
68
|
ma.cross_project_visibility AS visibility, ma.created_at
|
|
12
69
|
FROM memory_artifacts ma
|
|
13
70
|
WHERE ma.project_id = ?
|
|
14
|
-
AND ma.cross_project_visibility IN ${
|
|
71
|
+
AND ma.cross_project_visibility IN ${visList}
|
|
15
72
|
AND (ma.problem LIKE '%' || ? || '%' OR ma.validated_fix LIKE '%' || ? || '%')
|
|
16
73
|
ORDER BY ma.created_at DESC
|
|
17
74
|
LIMIT ?
|
|
18
|
-
`).all(
|
|
75
|
+
`).all(project.id, safeQ, safeQ, limit);
|
|
76
|
+
for (const r of rows) {
|
|
77
|
+
hits.push({
|
|
78
|
+
projectId: project.id,
|
|
79
|
+
projectName: project.name,
|
|
80
|
+
kind: 'artifact',
|
|
81
|
+
id: r.id,
|
|
82
|
+
title: r.title.slice(0, 100),
|
|
83
|
+
snippet: (r.snippet || '').slice(0, 200),
|
|
84
|
+
visibility: r.visibility,
|
|
85
|
+
capturedAt: r.created_at,
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
catch { /* tolerant */ }
|
|
90
|
+
}
|
|
91
|
+
// quests
|
|
92
|
+
if (kinds.has('quest')) {
|
|
93
|
+
try {
|
|
94
|
+
const rows = db.prepare(`
|
|
19
95
|
SELECT id, title, description, cross_project_visibility AS visibility, created_at
|
|
20
96
|
FROM quests
|
|
21
97
|
WHERE project_id = ?
|
|
22
|
-
AND cross_project_visibility IN ${
|
|
98
|
+
AND cross_project_visibility IN ${visList}
|
|
23
99
|
AND (title LIKE '%' || ? || '%' OR description LIKE '%' || ? || '%')
|
|
24
100
|
ORDER BY created_at DESC
|
|
25
101
|
LIMIT ?
|
|
26
|
-
`).all(
|
|
102
|
+
`).all(project.id, safeQ, safeQ, limit);
|
|
103
|
+
for (const r of rows) {
|
|
104
|
+
hits.push({
|
|
105
|
+
projectId: project.id,
|
|
106
|
+
projectName: project.name,
|
|
107
|
+
kind: 'quest',
|
|
108
|
+
id: r.id,
|
|
109
|
+
title: r.title,
|
|
110
|
+
snippet: (r.description || '').slice(0, 200),
|
|
111
|
+
visibility: r.visibility,
|
|
112
|
+
capturedAt: r.created_at,
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
catch { /* tolerant */ }
|
|
117
|
+
}
|
|
118
|
+
// design references (using existing FTS5 index)
|
|
119
|
+
if (kinds.has('reference')) {
|
|
120
|
+
try {
|
|
121
|
+
const rows = db.prepare(`
|
|
27
122
|
SELECT dr.id, dr.title, dr.notes, dr.tags, dr.cross_project_visibility AS visibility, dr.captured_at
|
|
28
123
|
FROM design_references dr
|
|
29
124
|
WHERE dr.project_id = ?
|
|
30
|
-
AND dr.cross_project_visibility IN ${
|
|
125
|
+
AND dr.cross_project_visibility IN ${visList}
|
|
31
126
|
AND (dr.title LIKE '%' || ? || '%' OR dr.notes LIKE '%' || ? || '%' OR dr.tags LIKE '%' || ? || '%')
|
|
32
127
|
ORDER BY dr.captured_at DESC
|
|
33
128
|
LIMIT ?
|
|
34
|
-
`).all(
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
129
|
+
`).all(project.id, safeQ, safeQ, safeQ, limit);
|
|
130
|
+
for (const r of rows) {
|
|
131
|
+
hits.push({
|
|
132
|
+
projectId: project.id,
|
|
133
|
+
projectName: project.name,
|
|
134
|
+
kind: 'reference',
|
|
135
|
+
id: r.id,
|
|
136
|
+
title: r.title || `(reference #${r.id})`,
|
|
137
|
+
snippet: [r.notes, r.tags].filter(Boolean).join(' · ').slice(0, 200),
|
|
138
|
+
visibility: r.visibility,
|
|
139
|
+
capturedAt: r.captured_at,
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
catch { /* tolerant */ }
|
|
144
|
+
}
|
|
145
|
+
return hits;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Run a constellation query across every project.
|
|
149
|
+
* Tolerant of older schemas (any table or column missing simply skips).
|
|
150
|
+
*/
|
|
151
|
+
export function constellationQuery(db, opts) {
|
|
152
|
+
const kinds = new Set((opts.kinds ?? ['truth', 'artifact', 'quest', 'reference']).filter((k) => SAFE_KINDS.has(k)));
|
|
153
|
+
const visibility = new Set((opts.visibility ?? ['org', 'public']).filter((v) => SAFE_VIS.has(v)));
|
|
154
|
+
// If the caller wants their OWN project's data too, allow 'within'.
|
|
155
|
+
// Otherwise we strictly enforce the cross-project gate.
|
|
156
|
+
const includeOwn = visibility.has('within');
|
|
157
|
+
const perProject = Math.min(Math.max(1, opts.perProjectLimit ?? 5), 50);
|
|
158
|
+
const projects = db.prepare(`SELECT id, name FROM projects ORDER BY id`).all();
|
|
159
|
+
const allHits = [];
|
|
160
|
+
for (const p of projects) {
|
|
161
|
+
if (!includeOwn && opts.callerProjectId && p.id === opts.callerProjectId)
|
|
162
|
+
continue;
|
|
163
|
+
const hits = queryProject(db, p, opts.query, kinds, visibility, perProject);
|
|
164
|
+
allHits.push(...hits);
|
|
165
|
+
}
|
|
166
|
+
// Sort: most recent first overall.
|
|
167
|
+
allHits.sort((a, b) => b.capturedAt.localeCompare(a.capturedAt));
|
|
168
|
+
return allHits;
|
|
169
|
+
}
|
|
170
|
+
export function renderConstellation(hits, query) {
|
|
171
|
+
if (hits.length === 0) {
|
|
172
|
+
return ` No constellation matches for *${query}* across registered projects.\n\n_Set \`cross_project_visibility\` to \`'org'\` or \`'public'\` on the items you want surfaced here. Default is \`'within'\` — within-project only._`;
|
|
173
|
+
}
|
|
174
|
+
// Group by project
|
|
175
|
+
const byProject = new Map();
|
|
176
|
+
for (const h of hits) {
|
|
177
|
+
const arr = byProject.get(h.projectName) ?? [];
|
|
178
|
+
arr.push(h);
|
|
179
|
+
byProject.set(h.projectName, arr);
|
|
180
|
+
}
|
|
181
|
+
const lines = [` **Constellation** — ${hits.length} match(es) for *${query}* across ${byProject.size} project(s)`];
|
|
182
|
+
for (const [name, items] of byProject) {
|
|
183
|
+
lines.push('');
|
|
184
|
+
lines.push(`### ${name}`);
|
|
185
|
+
for (const h of items) {
|
|
186
|
+
lines.push(`- \`${h.kind}:${h.id}\` **${h.title}** (${h.visibility})`);
|
|
187
|
+
if (h.snippet)
|
|
188
|
+
lines.push(` ${h.snippet}`);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
return lines.join('\n');
|
|
192
|
+
}
|
|
193
|
+
//# sourceMappingURL=constellation.js.map
|
|
@@ -1 +1,45 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Content-signature — the matching foundation of the Failure Firewall content
|
|
3
|
+
* tier (Failure Firewall v2). When a failure records the code that broke, the
|
|
4
|
+
* guard can later recognize an Edit/Write that RE-INTRODUCES it — even after the
|
|
5
|
+
* model reformatted whitespace or re-commented it — and block the repeat.
|
|
6
|
+
*
|
|
7
|
+
* Two pure helpers:
|
|
8
|
+
* - normalizeContentSignature: a whitespace/comment-insensitive normal form.
|
|
9
|
+
* - containsFailedSignature: a conservative match WITH a precision gate — a
|
|
10
|
+
* signature shorter than `minChars` never matches, so the firewall can never
|
|
11
|
+
* false-positive on a trivial fragment (the recon's hard requirement: a
|
|
12
|
+
* content firewall that false-positives gets ripped out).
|
|
13
|
+
*
|
|
14
|
+
* Deterministic + pure + zero-dep. No AST parse (language-agnostic); a future
|
|
15
|
+
* tier can layer a per-language AST normalizer behind the same interface.
|
|
16
|
+
*
|
|
17
|
+
* @copyright 2026 Ghost Protocol (Pvt) Ltd.
|
|
18
|
+
* @license Proprietary — (c) 2026 Ghost Protocol (Pvt) Ltd. All rights reserved. See LICENSE.
|
|
19
|
+
*/
|
|
20
|
+
/**
|
|
21
|
+
* Normalize a code edit to a comparable signature: strip comments, collapse all
|
|
22
|
+
* whitespace runs to a single space, trim. Trivial reformatting (indentation,
|
|
23
|
+
* line breaks, comments) no longer hides a re-introduced pattern.
|
|
24
|
+
*/
|
|
25
|
+
export function normalizeContentSignature(code) {
|
|
26
|
+
return code
|
|
27
|
+
.replace(/\/\*[\s\S]*?\*\//g, ' ') // /* block comments */
|
|
28
|
+
.replace(/(^|[^:])\/\/[^\n]*/gm, '$1') // // line comments (keep ':' so URLs survive)
|
|
29
|
+
.replace(/#[^\n]*/g, ' ') // # line comments (python/shell)
|
|
30
|
+
.replace(/\s+/g, ' ')
|
|
31
|
+
.trim();
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* True iff `editContent` re-introduces `failedSignature` (after normalization),
|
|
35
|
+
* AND the normalized signature is at least `minChars` long. The length gate is
|
|
36
|
+
* the precision floor: a recorded failure whose code fingerprint is shorter than
|
|
37
|
+
* `minChars` is too generic to block on without risking false positives.
|
|
38
|
+
*/
|
|
39
|
+
export function containsFailedSignature(editContent, failedSignature, minChars = 24) {
|
|
40
|
+
const needle = normalizeContentSignature(failedSignature);
|
|
41
|
+
if (needle.length < minChars)
|
|
42
|
+
return false;
|
|
43
|
+
return normalizeContentSignature(editContent).includes(needle);
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=content-signature.js.map
|