pico-engine 1.9.5 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. package/README.md +11 -5
  2. package/demos/bootstrap/com.example.visitors.bootstrap.krl +9 -5
  3. package/demos/child_tasks/com.example.task_runner.bootstrap.krl +10 -7
  4. package/demos/echo/com.example.echo.bootstrap.krl +10 -7
  5. package/demos/guard_rules/com.example.archive.bootstrap.krl +10 -7
  6. package/demos/json_query/README.md +11 -0
  7. package/demos/json_query/com.example.json_query.bootstrap.krl +84 -0
  8. package/demos/json_query/com.example.json_query.krl +162 -0
  9. package/demos/json_query/try-it.md +5 -0
  10. package/demos/kblog/com.example.kblog.bootstrap.krl +10 -7
  11. package/demos/layered_rulesets/com.example.invitations.bootstrap.krl +10 -7
  12. package/demos/score_tracker/com.example.score_tracker.bootstrap.krl +10 -7
  13. package/demos/secrets/com.example.notifier.bootstrap.krl +10 -7
  14. package/demos/try-it-order.json +1 -1
  15. package/demos/two_picos/com.example.two_picos.bootstrap.krl +5 -2
  16. package/demos/xmlui/com.example.thermostat.bootstrap.krl +10 -7
  17. package/dist/cli.js +5 -1
  18. package/dist/engineConfig.js +1 -1
  19. package/dist/identity/portableMove.js +5 -1
  20. package/dist/index.js +44 -12
  21. package/dist/internalAccess.js +110 -0
  22. package/dist/oauth/OAuthService.js +172 -10
  23. package/dist/oauth/acg.js +194 -24
  24. package/dist/oauth/cimd.js +381 -0
  25. package/dist/oauth/discovery.js +175 -0
  26. package/dist/runMcp.js +1 -1
  27. package/dist/server.js +197 -48
  28. package/dist/subtreeWrangler.js +8 -5
  29. package/dist/uiChannelPolicies.js +2 -0
  30. package/krl/io.picolabs.pds.krl +109 -21
  31. package/krl/io.picolabs.pico-engine-ui.krl +13 -7
  32. package/krl/io.picolabs.wrangler.krl +21 -9
  33. package/mcp/devResources.js +52 -0
  34. package/mcp/devTools.js +40 -9
  35. package/mcp/docs/CLAUDE.md +3 -2
  36. package/mcp/docs/cookbook/README.md +23 -0
  37. package/mcp/docs/cookbook/bootstrap/README.md +23 -0
  38. package/mcp/docs/cookbook/bootstrap/instructions.md +75 -0
  39. package/mcp/docs/cookbook/child_tasks/README.md +20 -0
  40. package/mcp/docs/cookbook/child_tasks/instructions.md +60 -0
  41. package/mcp/docs/cookbook/collection/README.md +21 -0
  42. package/mcp/docs/cookbook/collection/instructions.md +93 -0
  43. package/mcp/docs/cookbook/echo/README.md +19 -0
  44. package/mcp/docs/cookbook/echo/instructions.md +59 -0
  45. package/mcp/docs/cookbook/guard_rules/README.md +22 -0
  46. package/mcp/docs/cookbook/guard_rules/instructions.md +73 -0
  47. package/mcp/docs/cookbook/json_query/README.md +26 -0
  48. package/mcp/docs/cookbook/json_query/instructions.md +64 -0
  49. package/mcp/docs/cookbook/kblog/README.md +60 -0
  50. package/mcp/docs/cookbook/kblog/instructions.md +102 -0
  51. package/mcp/docs/cookbook/layered_rulesets/README.md +24 -0
  52. package/mcp/docs/cookbook/layered_rulesets/instructions.md +87 -0
  53. package/mcp/docs/cookbook/moving_picos/README.md +44 -0
  54. package/mcp/docs/cookbook/moving_picos/instructions.md +150 -0
  55. package/mcp/docs/cookbook/score_tracker/README.md +19 -0
  56. package/mcp/docs/cookbook/score_tracker/instructions.md +73 -0
  57. package/mcp/docs/cookbook/secrets/README.md +25 -0
  58. package/mcp/docs/cookbook/secrets/instructions.md +98 -0
  59. package/mcp/docs/cookbook/two_picos/README.md +20 -0
  60. package/mcp/docs/cookbook/two_picos/instructions.md +88 -0
  61. package/mcp/docs/cookbook/xmlui/README.md +23 -0
  62. package/mcp/docs/cookbook/xmlui/instructions.md +84 -0
  63. package/mcp/docs/developing/README.md +28 -0
  64. package/mcp/docs/developing/accepting-relationship-requests.md +65 -0
  65. package/mcp/docs/developing/arrays-and-maps.md +77 -0
  66. package/mcp/docs/developing/converting-time-formats.md +28 -0
  67. package/mcp/docs/developing/cookies-and-redirects.md +81 -0
  68. package/mcp/docs/developing/decoding-form-data.md +67 -0
  69. package/mcp/docs/developing/engine-log-files.md +34 -0
  70. package/mcp/docs/developing/event-loop.md +77 -0
  71. package/mcp/docs/developing/events-from-email.md +80 -0
  72. package/mcp/docs/developing/initializing-entity-variables.md +83 -0
  73. package/mcp/docs/developing/mcp-ai-tools.md +98 -0
  74. package/mcp/docs/developing/misconceptions.md +467 -0
  75. package/mcp/docs/developing/names-and-scope.md +44 -0
  76. package/mcp/docs/developing/postman.md +159 -0
  77. package/mcp/docs/developing/querying-another-pico.md +47 -0
  78. package/mcp/docs/developing/singleton-picos.md +36 -0
  79. package/mcp/docs/developing/tabular-data.md +67 -0
  80. package/mcp/docs/developing/tips.md +251 -0
  81. package/mcp/docs/developing/webhook-signatures.md +56 -0
  82. package/mcp/docs/developing/workflow.md +106 -0
  83. package/mcp/docs/exp/README.md +46 -0
  84. package/mcp/docs/exp/event.md +110 -0
  85. package/mcp/docs/exp/query.md +94 -0
  86. package/mcp/docs/lessons/README.md +59 -0
  87. package/mcp/docs/lessons/events-and-queries.md +247 -0
  88. package/mcp/docs/lessons/gossip.md +28 -0
  89. package/mcp/docs/lessons/modules-and-apis.md +270 -0
  90. package/mcp/docs/lessons/pico-based-systems.md +470 -0
  91. package/mcp/docs/lessons/pico-state.md +267 -0
  92. package/mcp/docs/lessons/pico-views.md +290 -0
  93. package/mcp/docs/lessons/quickstart.md +157 -0
  94. package/mcp/docs/lessons/relationships.md +530 -0
  95. package/mcp/docs/lessons/scatter-gather.md +362 -0
  96. package/mcp/docs/pico-engine.md +7 -5
  97. package/mcp/docs/pico-overview.md +3 -3
  98. package/mcp/docs/wrangler/README.md +47 -0
  99. package/mcp/docs/wrangler/channels.md +235 -0
  100. package/mcp/docs/wrangler/oauth.md +56 -0
  101. package/mcp/docs/wrangler/pds.md +423 -0
  102. package/mcp/docs/wrangler/portable-subtree.md +44 -0
  103. package/mcp/docs/wrangler/relationships.md +297 -0
  104. package/mcp/docs/wrangler/wrangler.md +253 -0
  105. package/mcp/docs/wrangler/xmlui.md +60 -0
  106. package/mcp/engineClient.js +57 -5
  107. package/mcp/mcpScopes.js +118 -0
  108. package/mcp/meshHttp.js +151 -45
  109. package/mcp/runStdio.js +2 -0
  110. package/mcp/subtreeTools.js +184 -0
  111. package/mcp/tryItTools.js +202 -0
  112. package/package.json +14 -8
  113. package/public/pico-engine-ui.js +26 -26
  114. package/src/cli.ts +6 -2
  115. package/src/engineConfig.ts +5 -2
  116. package/src/identity/portableMove.ts +6 -1
  117. package/src/index.ts +469 -430
  118. package/src/internalAccess.ts +127 -0
  119. package/src/oauth/OAuthService.ts +243 -11
  120. package/src/oauth/acg.ts +289 -25
  121. package/src/oauth/cimd.ts +508 -0
  122. package/src/oauth/discovery.ts +233 -0
  123. package/src/runMcp.ts +3 -1
  124. package/src/server.ts +252 -52
  125. package/src/subtreeWrangler.ts +17 -5
  126. package/src/uiChannelPolicies.ts +2 -0
package/src/index.ts CHANGED
@@ -1,430 +1,469 @@
1
- import { krl, KrlLogger, makeKrlLogger } from "krl-stdlib";
2
- import * as _ from "lodash";
3
- import * as makeDir from "make-dir";
4
- import * as path from "path";
5
- import type { Server } from "http";
6
- import { PicoEngineCore, RulesetRegistry } from "pico-engine-core";
7
- import {
8
- EventPolicyRule,
9
- PicoDbKey,
10
- PicoFramework,
11
- PolicyCeiling,
12
- QueryPolicyRule,
13
- } from "pico-framework";
14
- import { AuthService, WebAuthnAdapter, makeWebAuthnAdapter } from "./auth";
15
- import {
16
- IdentityStore,
17
- IdentityService,
18
- initDidoModule,
19
- createSubtreePortableHooks,
20
- } from "./identity";
21
- import { registerWebvhRoutes } from "./identity/webvhRoutes";
22
- import { OAuthService, initOAuthModule } from "./oauth";
23
- import {
24
- picoLogEntries,
25
- makeEngineLogWriter,
26
- resolveEngineLogFilePath,
27
- resolvePicoLogsReadPath,
28
- } from "./logging";
29
- import { openClassicLevelWithRecovery } from "./levelDb";
30
- import { provisionRoot, uiECIForRoot } from "./provisionRoot";
31
- import { createRulesetRegistryLoaderFs } from "./RulesetRegistryLoaderFs";
32
- import { resolveBundledKrlUrl } from "./bundledKrl";
33
- import { wranglerSubtreeImportServices } from "./subtreeWrangler";
34
- import { server } from "./server";
35
- import {
36
- installEngineRuntimeSettings,
37
- resolveEngineSettings,
38
- ShowTryItMenu,
39
- } from "./engineConfig";
40
- const charwise = require("charwise");
41
- const safeJsonCodec = require("level-json-coerce-null");
42
-
43
- const version = require("../package.json").version;
44
-
45
- /**
46
- * Configuration options that may be set by the user
47
- */
48
- export interface PicoEngineConfiguration {
49
- /**
50
- * The absolute path to the folder where the engine should store the database and logs.
51
- *
52
- * Default: "~/.pico-engine/"
53
- */
54
- home?: string;
55
-
56
- /**
57
- * The port number the http server should listen on.
58
- *
59
- * If you want an available port assigned to you, set it to 0
60
- *
61
- * Default: 3000
62
- */
63
- port?: number;
64
-
65
- /**
66
- * The base url others should use when addressing your engine.
67
- *
68
- * Default: "http://localhost:3000"
69
- */
70
- base_url?: string;
71
-
72
- /**
73
- * Provide any custom krl modules
74
- */
75
- modules?: { [domain: string]: krl.Module };
76
-
77
- /**
78
- * Trust event.time input. Used for testing
79
- */
80
- useEventInputTime?: boolean;
81
-
82
- log?: KrlLogger;
83
-
84
- /**
85
- * WebAuthn relying-party id (a registrable domain). Defaults to the hostname
86
- * of `base_url` (e.g. "localhost").
87
- */
88
- rpID?: string;
89
-
90
- /**
91
- * Human-friendly relying-party name shown by the OS passkey UI.
92
- */
93
- rpName?: string;
94
-
95
- /**
96
- * Expected WebAuthn origin (scheme://host[:port]). Defaults to the origin of
97
- * `base_url`.
98
- */
99
- origin?: string;
100
-
101
- /**
102
- * When true, anyone may register additional accounts/roots after bootstrap.
103
- * When false (default), only bootstrap (zero accounts), legacy claim, or a valid
104
- * invite may create a new account.
105
- */
106
- allowSelfSignup?: boolean;
107
-
108
- /**
109
- * Where the developer UI shows the Try It! tab: "off", "root_only" (default),
110
- * or "all". Config file key: show_tryit_menu.
111
- */
112
- showTryItMenu?: ShowTryItMenu;
113
-
114
- /**
115
- * Override the WebAuthn ceremony/verification implementation. Intended for
116
- * testing with a deterministic authenticator.
117
- */
118
- webauthn?: WebAuthnAdapter;
119
-
120
- /**
121
- * When true, boot auto-creates a single root (historical behavior). Defaults
122
- * to false — fresh engines start with zero roots until passkey registration.
123
- * Tests may set this to true.
124
- */
125
- autoCreateRootPico?: boolean;
126
-
127
- /** Default max channel policy for root picos (1.8 policy ceiling). */
128
- defaultRootPolicyCeiling?: PolicyCeiling;
129
-
130
- /** Engine-wide event denies on every channel ECI (1.8). */
131
- globalEventDeny?: EventPolicyRule[];
132
-
133
- /** Engine-wide query denies on every channel ECI (1.8). */
134
- globalQueryDeny?: QueryPolicyRule[];
135
- }
136
-
137
- export interface PicoEngine {
138
- version: string;
139
-
140
- home: string;
141
- port: number;
142
- base_url: string;
143
-
144
- pf: PicoFramework;
145
- core: PicoEngineCore;
146
- /** UI channel for the session root; null when no roots exist yet at boot. */
147
- uiECI: string | null;
148
- rsRegistry: RulesetRegistry;
149
- auth: AuthService;
150
- oauth: OAuthService;
151
- identity: IdentityStore;
152
- identityService: IdentityService;
153
-
154
- /** Stop HTTP and close LevelDB stores (for container SIGTERM). */
155
- shutdown(): Promise<void>;
156
- }
157
-
158
- /** Integration harnesses (e.g. manifold-api Docker tests): boot with one root, no passkey. */
159
- function autoCreateRootFromEnv(): boolean | undefined {
160
- const v = process.env.PICO_ENGINE_AUTO_CREATE_ROOT;
161
- if (typeof v !== "string" || v.trim().length === 0) {
162
- return undefined;
163
- }
164
- const s = v.trim().toLowerCase();
165
- if (s === "1" || s === "true" || s === "yes") {
166
- return true;
167
- }
168
- if (s === "0" || s === "false" || s === "no") {
169
- return false;
170
- }
171
- return undefined;
172
- }
173
-
174
- export async function startEngine(
175
- configuration: PicoEngineConfiguration = {}
176
- ): Promise<PicoEngine> {
177
- const settings = resolveEngineSettings({
178
- home: configuration.home,
179
- port: configuration.port,
180
- base_url: configuration.base_url,
181
- rpID: configuration.rpID,
182
- rpName: configuration.rpName,
183
- origin: configuration.origin,
184
- allowSelfSignup: configuration.allowSelfSignup,
185
- showTryItMenu: configuration.showTryItMenu,
186
- defaultRootPolicyCeiling: configuration.defaultRootPolicyCeiling,
187
- globalEventDeny: configuration.globalEventDeny,
188
- globalQueryDeny: configuration.globalQueryDeny,
189
- });
190
- installEngineRuntimeSettings(settings);
191
-
192
- const home = settings.home;
193
- let port = configuration.port ?? settings.port;
194
- const baseUrlFromCaller = configuration.base_url;
195
- /** Filled after listen when using port 0 or when config baseUrl port does not match. */
196
- let base_url: string | undefined = baseUrlFromCaller;
197
- if (base_url == null && port > 0) {
198
- base_url = settings.baseUrl;
199
- }
200
- await makeDir(home);
201
-
202
- const logFilePath = resolveEngineLogFilePath(home);
203
- const log = configuration.log
204
- ? configuration.log
205
- : makeKrlLogger(makeEngineLogWriter(logFilePath, home));
206
-
207
- const db = await openClassicLevelWithRecovery<PicoDbKey, any>(
208
- path.resolve(home, "db"),
209
- {
210
- keyEncoding: charwise,
211
- valueEncoding: safeJsonCodec,
212
- },
213
- log
214
- );
215
- const rulesetLoaderHandle = await createRulesetRegistryLoaderFs(home, log);
216
-
217
- const core = new PicoEngineCore({
218
- db,
219
- rsRegLoader: rulesetLoaderHandle.loader,
220
- log,
221
- modules: configuration.modules,
222
- useEventInputTime: configuration.useEventInputTime,
223
- autoCreateRootPico:
224
- configuration.autoCreateRootPico ??
225
- autoCreateRootFromEnv() ??
226
- false,
227
- defaultRootPolicyCeiling:
228
- configuration.defaultRootPolicyCeiling ??
229
- settings.defaultRootPolicyCeiling,
230
- globalEventDeny: configuration.globalEventDeny ?? settings.globalEventDeny,
231
- globalQueryDeny: configuration.globalQueryDeny ?? settings.globalQueryDeny,
232
- getPicoLogs(picoId) {
233
- return Promise.resolve(picoLogEntries(picoId));
234
- },
235
- });
236
- const pf = core.picoFramework;
237
- core.resolveBundledRulesetUrl = resolveBundledKrlUrl;
238
- core.subtreeImportServices = wranglerSubtreeImportServices(core.rsRegistry);
239
- const identity = new IdentityStore({ db: pf.db });
240
- let identityService!: IdentityService;
241
- const onPicoCreated = async (
242
- picoId: string,
243
- ctx: { isRoot: boolean; parentPicoId: string }
244
- ) => {
245
- if (core.base_url) {
246
- await identityService.ensureWebvhDid(picoId, { isRoot: ctx.isRoot });
247
- }
248
- };
249
- identityService = new IdentityService({
250
- store: identity,
251
- getBaseUrl: () => core.base_url || base_url || "",
252
- pf,
253
- rsRegistry: core.rsRegistry,
254
- resolveBundledRulesetUrl: resolveBundledKrlUrl,
255
- reloadImportedSchedules: (picoIds) => core.reloadImportedSchedules(picoIds),
256
- firePicoImported: async (result) => {
257
- const importedRoot = pf.getPico(result.newRootPicoId);
258
- if (!importedRoot) {
259
- return;
260
- }
261
- await importedRoot.event({
262
- eci: result.newRootEci,
263
- domain: "wrangler",
264
- name: "pico_imported",
265
- data: {
266
- attrs: {
267
- oldRootPicoId: result.oldRootPicoId,
268
- newRootPicoId: result.newRootPicoId,
269
- eciMap: result.eciMap,
270
- picoIdMap: result.picoIdMap,
271
- portableIdentity: true,
272
- },
273
- },
274
- time: Date.now(),
275
- });
276
- },
277
- });
278
- core.onPicoCreated = onPicoCreated;
279
- core.subtreePortableHooks = createSubtreePortableHooks(identity, pf);
280
- core.modules["dido"] = initDidoModule(identityService);
281
-
282
- try {
283
- await core.start();
284
- } catch (err) {
285
- log.error("Engine startup failed", { error: err });
286
- await rulesetLoaderHandle.close().catch(() => undefined);
287
- await db.close().catch(() => undefined);
288
- throw err;
289
- }
290
- const rsRegistry = core.rsRegistry;
291
-
292
- // Fresh engines boot with zero roots. Adopt + (re)provision only when roots
293
- // already exist (test harness with autoCreateRootPico, or a migrated DB).
294
- let uiECI: string | null = null;
295
- const existingRoots = pf.rootPicos();
296
- if (existingRoots.length > 0) {
297
- const provisioned = await provisionRoot(pf, core, { root: existingRoots[0] });
298
- uiECI = provisioned.uiECI;
299
- }
300
-
301
- const auth = new AuthService({
302
- db: pf.db,
303
- webauthn: configuration.webauthn || makeWebAuthnAdapter(),
304
- getBaseUrl: () => core.base_url,
305
- rpID: configuration.rpID,
306
- rpName: configuration.rpName,
307
- origin: configuration.origin,
308
- allowSelfSignup:
309
- configuration.allowSelfSignup ?? settings.allowSelfSignup,
310
- flushBootstrapUrl: async (url) => {
311
- const flushed = await rsRegistry.flush(url);
312
- return { rid: flushed.rid };
313
- },
314
- provisionRoot: async (opts) => {
315
- const { root, uiECI } = await provisionRoot(pf, core, opts);
316
- if (core.base_url) {
317
- await identityService.ensureWebvhDid(root.id, { isRoot: true });
318
- }
319
- return { rootPicoId: root.id, uiECI };
320
- },
321
- getUiECI: (rootPicoId: string) => uiECIForRoot(pf, rootPicoId),
322
- getPrimaryRootPicoId: () => {
323
- try {
324
- return pf.rootPico.id;
325
- } catch {
326
- return null;
327
- }
328
- },
329
- onAccountClaimed: async (rootPicoId, displayName) => {
330
- const uiChannel = uiECIForRoot(pf, rootPicoId);
331
- if (!uiChannel || !displayName.trim()) {
332
- return;
333
- }
334
- await pf.eventWait({
335
- eci: uiChannel,
336
- domain: "engine_ui",
337
- name: "box",
338
- data: { attrs: { name: displayName.trim() } },
339
- time: 0,
340
- });
341
- },
342
- });
343
-
344
- const oauth = new OAuthService({
345
- db: pf.db,
346
- pf,
347
- });
348
- core.modules["oauth"] = initOAuthModule(oauth);
349
-
350
- const app = server(core, uiECI, auth, oauth, identityService);
351
-
352
- if ((!port || !_.isInteger(port) || port < 1) && port !== 0) {
353
- port = process.env.NODE_ENV === "test" ? 0 : 3000;
354
- }
355
- let httpServer: Server | undefined;
356
- await new Promise<void>((resolve) => {
357
- httpServer = app.listen(port, () => {
358
- if (httpServer) {
359
- const addr = httpServer.address();
360
- if (addr && typeof addr !== "string" && _.isInteger(addr.port)) {
361
- // Get the actual port i.e. if they set port to 0 nodejs will assign you an available port
362
- port = addr.port;
363
- }
364
- }
365
- resolve(undefined);
366
- });
367
- });
368
- if (typeof base_url !== "string") {
369
- base_url = `http://localhost:${port}`;
370
- } else if (
371
- baseUrlFromCaller == null &&
372
- (configuration.port === 0 || port !== settings.port)
373
- ) {
374
- base_url = `http://localhost:${port}`;
375
- }
376
- core.base_url = base_url;
377
-
378
- await identityService.ensureWebvhForAllLoadedPicos();
379
-
380
- log.info(`Listening at ${base_url}`);
381
-
382
- if (uiECI) {
383
- pf.event({
384
- eci: uiECI,
385
- domain: "engine",
386
- name: "started",
387
- data: { attrs: {} },
388
- time: 0, // TODO remove this typescript requirement
389
- }).catch((error) => {
390
- log.error("Error signaling engine:started event", { error });
391
- });
392
- }
393
-
394
- let shuttingDown = false;
395
- async function shutdown() {
396
- if (shuttingDown) {
397
- return;
398
- }
399
- shuttingDown = true;
400
- log.info("Shutting down pico-engine");
401
- await new Promise<void>((resolve, reject) => {
402
- if (!httpServer) {
403
- resolve(undefined);
404
- return;
405
- }
406
- httpServer.close((err) => (err ? reject(err) : resolve(undefined)));
407
- });
408
- await rulesetLoaderHandle.close();
409
- await db.close();
410
- log.info("Shutdown complete");
411
- }
412
-
413
- return {
414
- version,
415
-
416
- home,
417
- port,
418
- base_url,
419
-
420
- pf,
421
- core,
422
- uiECI,
423
- rsRegistry,
424
- auth,
425
- oauth,
426
- identity,
427
- identityService,
428
- shutdown,
429
- };
430
- }
1
+ import { krl, KrlLogger, makeKrlLogger } from "krl-stdlib";
2
+ import * as _ from "lodash";
3
+ import * as makeDir from "make-dir";
4
+ import * as path from "path";
5
+ import type { Server } from "http";
6
+ import { PicoEngineCore, RulesetRegistry, warmPdsSecretLog } from "pico-engine-core";
7
+ import {
8
+ EventPolicyRule,
9
+ PicoDbKey,
10
+ PicoFramework,
11
+ PolicyCeiling,
12
+ QueryPolicyRule,
13
+ } from "pico-framework";
14
+ import { AuthService, WebAuthnAdapter, makeWebAuthnAdapter } from "./auth";
15
+ import {
16
+ IdentityStore,
17
+ IdentityService,
18
+ initDidoModule,
19
+ createSubtreePortableHooks,
20
+ } from "./identity";
21
+ import { registerWebvhRoutes } from "./identity/webvhRoutes";
22
+ import { OAuthService, initOAuthModule } from "./oauth";
23
+ import {
24
+ picoLogEntries,
25
+ makeEngineLogWriter,
26
+ resolveEngineLogFilePath,
27
+ resolvePicoLogsReadPath,
28
+ } from "./logging";
29
+ import { openClassicLevelWithRecovery } from "./levelDb";
30
+ import { provisionRoot, uiECIForRoot } from "./provisionRoot";
31
+ import { createRulesetRegistryLoaderFs } from "./RulesetRegistryLoaderFs";
32
+ import { resolveBundledKrlUrl } from "./bundledKrl";
33
+ import { wranglerSubtreeImportServices } from "./subtreeWrangler";
34
+ import { writeInternalToken } from "./internalAccess";
35
+ import { server } from "./server";
36
+ import {
37
+ installEngineRuntimeSettings,
38
+ resolveEngineSettings,
39
+ ShowTryItMenu,
40
+ } from "./engineConfig";
41
+ const charwise = require("charwise");
42
+ const safeJsonCodec = require("level-json-coerce-null");
43
+
44
+ const version = require("../package.json").version;
45
+
46
+ /**
47
+ * Configuration options that may be set by the user
48
+ */
49
+ export interface PicoEngineConfiguration {
50
+ /**
51
+ * The absolute path to the folder where the engine should store the database and logs.
52
+ *
53
+ * Default: "~/.pico-engine/"
54
+ */
55
+ home?: string;
56
+
57
+ /**
58
+ * The port number the http server should listen on.
59
+ *
60
+ * If you want an available port assigned to you, set it to 0
61
+ *
62
+ * Default: 3000
63
+ */
64
+ port?: number;
65
+
66
+ /**
67
+ * The base url others should use when addressing your engine.
68
+ *
69
+ * Default: "http://localhost:3000"
70
+ */
71
+ base_url?: string;
72
+
73
+ /**
74
+ * Provide any custom krl modules
75
+ */
76
+ modules?: { [domain: string]: krl.Module };
77
+
78
+ /**
79
+ * Trust event.time input. Used for testing
80
+ */
81
+ useEventInputTime?: boolean;
82
+
83
+ log?: KrlLogger;
84
+
85
+ /**
86
+ * WebAuthn relying-party id (a registrable domain). Defaults to the hostname
87
+ * of `base_url` (e.g. "localhost").
88
+ */
89
+ rpID?: string;
90
+
91
+ /**
92
+ * Human-friendly relying-party name shown by the OS passkey UI.
93
+ */
94
+ rpName?: string;
95
+
96
+ /**
97
+ * Expected WebAuthn origin (scheme://host[:port]). Defaults to the origin of
98
+ * `base_url`.
99
+ */
100
+ origin?: string;
101
+
102
+ /**
103
+ * When true, anyone may register additional accounts/roots after bootstrap.
104
+ * When false (default), only bootstrap (zero accounts), legacy claim, or a valid
105
+ * invite may create a new account.
106
+ */
107
+ allowSelfSignup?: boolean;
108
+
109
+ /**
110
+ * When true, a direct loopback connection may call `/c/*`, `/api/dev/local`,
111
+ * and `/api/try-it` without a passkey or the internal token. Default false.
112
+ * Proxied requests and a spoofed Host header never qualify.
113
+ */
114
+ allowLocalhostChannelBypass?: boolean;
115
+
116
+ /**
117
+ * Where the developer UI shows the Try It! tab: "off", "root_only" (default),
118
+ * or "all". Config file key: show_tryit_menu.
119
+ */
120
+ showTryItMenu?: ShowTryItMenu;
121
+
122
+ /**
123
+ * Override the WebAuthn ceremony/verification implementation. Intended for
124
+ * testing with a deterministic authenticator.
125
+ */
126
+ webauthn?: WebAuthnAdapter;
127
+
128
+ /**
129
+ * When true, boot auto-creates a single root (historical behavior). Defaults
130
+ * to false — fresh engines start with zero roots until passkey registration.
131
+ * Tests may set this to true.
132
+ */
133
+ autoCreateRootPico?: boolean;
134
+
135
+ /** Default max channel policy for root picos (1.8 policy ceiling). */
136
+ defaultRootPolicyCeiling?: PolicyCeiling;
137
+
138
+ /** Engine-wide event denies on every channel ECI (1.8). */
139
+ globalEventDeny?: EventPolicyRule[];
140
+
141
+ /** Engine-wide query denies on every channel ECI (1.8). */
142
+ globalQueryDeny?: QueryPolicyRule[];
143
+ }
144
+
145
+ export interface PicoEngine {
146
+ version: string;
147
+
148
+ home: string;
149
+ port: number;
150
+ base_url: string;
151
+
152
+ pf: PicoFramework;
153
+ core: PicoEngineCore;
154
+ /** UI channel for the session root; null when no roots exist yet at boot. */
155
+ uiECI: string | null;
156
+ rsRegistry: RulesetRegistry;
157
+ auth: AuthService;
158
+ oauth: OAuthService;
159
+ identity: IdentityStore;
160
+ identityService: IdentityService;
161
+
162
+ /** Stop HTTP and close LevelDB stores (for container SIGTERM). */
163
+ shutdown(): Promise<void>;
164
+ }
165
+
166
+ /** Integration harnesses (e.g. manifold-api Docker tests): boot with one root, no passkey. */
167
+ function autoCreateRootFromEnv(): boolean | undefined {
168
+ const v = process.env.PICO_ENGINE_AUTO_CREATE_ROOT;
169
+ if (typeof v !== "string" || v.trim().length === 0) {
170
+ return undefined;
171
+ }
172
+ const s = v.trim().toLowerCase();
173
+ if (s === "1" || s === "true" || s === "yes") {
174
+ return true;
175
+ }
176
+ if (s === "0" || s === "false" || s === "no") {
177
+ return false;
178
+ }
179
+ return undefined;
180
+ }
181
+
182
+ export async function startEngine(
183
+ configuration: PicoEngineConfiguration = {}
184
+ ): Promise<PicoEngine> {
185
+ const settings = resolveEngineSettings({
186
+ home: configuration.home,
187
+ port: configuration.port,
188
+ base_url: configuration.base_url,
189
+ rpID: configuration.rpID,
190
+ rpName: configuration.rpName,
191
+ origin: configuration.origin,
192
+ allowSelfSignup: configuration.allowSelfSignup,
193
+ allowLocalhostChannelBypass: configuration.allowLocalhostChannelBypass,
194
+ showTryItMenu: configuration.showTryItMenu,
195
+ defaultRootPolicyCeiling: configuration.defaultRootPolicyCeiling,
196
+ globalEventDeny: configuration.globalEventDeny,
197
+ globalQueryDeny: configuration.globalQueryDeny,
198
+ });
199
+ installEngineRuntimeSettings(settings);
200
+
201
+ const home = settings.home;
202
+ let port = configuration.port ?? settings.port;
203
+ const baseUrlFromCaller = configuration.base_url;
204
+ /** Filled after listen when using port 0 or when config baseUrl port does not match. */
205
+ let base_url: string | undefined = baseUrlFromCaller;
206
+ if (base_url == null && port > 0) {
207
+ base_url = settings.baseUrl;
208
+ }
209
+ await makeDir(home);
210
+
211
+ const logFilePath = resolveEngineLogFilePath(home);
212
+ const log = configuration.log
213
+ ? configuration.log
214
+ : makeKrlLogger(makeEngineLogWriter(logFilePath, home));
215
+
216
+ const db = await openClassicLevelWithRecovery<PicoDbKey, any>(
217
+ path.resolve(home, "db"),
218
+ {
219
+ keyEncoding: charwise,
220
+ valueEncoding: safeJsonCodec,
221
+ },
222
+ log
223
+ );
224
+ const rulesetLoaderHandle = await createRulesetRegistryLoaderFs(home, log);
225
+
226
+ const core = new PicoEngineCore({
227
+ db,
228
+ rsRegLoader: rulesetLoaderHandle.loader,
229
+ log,
230
+ modules: configuration.modules,
231
+ useEventInputTime: configuration.useEventInputTime,
232
+ autoCreateRootPico:
233
+ configuration.autoCreateRootPico ??
234
+ autoCreateRootFromEnv() ??
235
+ false,
236
+ defaultRootPolicyCeiling:
237
+ configuration.defaultRootPolicyCeiling ??
238
+ settings.defaultRootPolicyCeiling,
239
+ globalEventDeny: configuration.globalEventDeny ?? settings.globalEventDeny,
240
+ globalQueryDeny: configuration.globalQueryDeny ?? settings.globalQueryDeny,
241
+ getPicoLogs(picoId) {
242
+ return Promise.resolve(picoLogEntries(picoId));
243
+ },
244
+ });
245
+ const pf = core.picoFramework;
246
+ core.resolveBundledRulesetUrl = resolveBundledKrlUrl;
247
+ core.subtreeImportServices = wranglerSubtreeImportServices(core.rsRegistry, (picoId, ctx) =>
248
+ onPicoCreated(picoId, ctx)
249
+ );
250
+ const identity = new IdentityStore({ db: pf.db });
251
+ let identityService!: IdentityService;
252
+ const onPicoCreated = async (
253
+ picoId: string,
254
+ ctx: { isRoot: boolean; parentPicoId: string }
255
+ ) => {
256
+ if (core.base_url) {
257
+ await identityService.ensureWebvhDid(picoId, { isRoot: ctx.isRoot });
258
+ }
259
+ };
260
+ identityService = new IdentityService({
261
+ store: identity,
262
+ getBaseUrl: () => core.base_url || base_url || "",
263
+ pf,
264
+ rsRegistry: core.rsRegistry,
265
+ resolveBundledRulesetUrl: resolveBundledKrlUrl,
266
+ reloadImportedSchedules: (picoIds) => core.reloadImportedSchedules(picoIds),
267
+ firePicoImported: async (result) => {
268
+ const importedRoot = pf.getPico(result.newRootPicoId);
269
+ if (!importedRoot) {
270
+ return;
271
+ }
272
+ await importedRoot.event({
273
+ eci: result.newRootEci,
274
+ domain: "wrangler",
275
+ name: "pico_imported",
276
+ data: {
277
+ attrs: {
278
+ oldRootPicoId: result.oldRootPicoId,
279
+ newRootPicoId: result.newRootPicoId,
280
+ eciMap: result.eciMap,
281
+ picoIdMap: result.picoIdMap,
282
+ portableIdentity: true,
283
+ },
284
+ },
285
+ time: Date.now(),
286
+ });
287
+ },
288
+ });
289
+ core.onPicoCreated = onPicoCreated;
290
+ core.subtreePortableHooks = createSubtreePortableHooks(identity, pf);
291
+ core.modules["dido"] = initDidoModule(identityService);
292
+
293
+ try {
294
+ await core.start();
295
+ } catch (err) {
296
+ log.error("Engine startup failed", { error: err });
297
+ await rulesetLoaderHandle.close().catch(() => undefined);
298
+ await db.close().catch(() => undefined);
299
+ throw err;
300
+ }
301
+ const rsRegistry = core.rsRegistry;
302
+
303
+ // Fresh engines boot with zero roots. Adopt + (re)provision only when roots
304
+ // already exist (test harness with autoCreateRootPico, or a migrated DB).
305
+ let uiECI: string | null = null;
306
+ const existingRoots = pf.rootPicos();
307
+ if (existingRoots.length > 0) {
308
+ const provisioned = await provisionRoot(pf, core, { root: existingRoots[0] });
309
+ uiECI = provisioned.uiECI;
310
+ }
311
+
312
+ const auth = new AuthService({
313
+ db: pf.db,
314
+ webauthn: configuration.webauthn || makeWebAuthnAdapter(),
315
+ getBaseUrl: () => core.base_url,
316
+ rpID: configuration.rpID,
317
+ rpName: configuration.rpName,
318
+ origin: configuration.origin,
319
+ allowSelfSignup:
320
+ configuration.allowSelfSignup ?? settings.allowSelfSignup,
321
+ flushBootstrapUrl: async (url) => {
322
+ const flushed = await rsRegistry.flush(url);
323
+ return { rid: flushed.rid };
324
+ },
325
+ provisionRoot: async (opts) => {
326
+ const { root, uiECI } = await provisionRoot(pf, core, opts);
327
+ if (core.base_url) {
328
+ await identityService.ensureWebvhDid(root.id, { isRoot: true });
329
+ }
330
+ return { rootPicoId: root.id, uiECI };
331
+ },
332
+ getUiECI: (rootPicoId: string) => uiECIForRoot(pf, rootPicoId),
333
+ getPrimaryRootPicoId: () => {
334
+ try {
335
+ return pf.rootPico.id;
336
+ } catch {
337
+ return null;
338
+ }
339
+ },
340
+ onAccountClaimed: async (rootPicoId, displayName) => {
341
+ const uiChannel = uiECIForRoot(pf, rootPicoId);
342
+ if (!uiChannel || !displayName.trim()) {
343
+ return;
344
+ }
345
+ await pf.eventWait({
346
+ eci: uiChannel,
347
+ domain: "engine_ui",
348
+ name: "box",
349
+ data: { attrs: { name: displayName.trim() } },
350
+ time: 0,
351
+ });
352
+ },
353
+ });
354
+
355
+ const oauth = new OAuthService({
356
+ db: pf.db,
357
+ pf,
358
+ getBaseUrl: () => core.base_url,
359
+ });
360
+ core.modules["oauth"] = initOAuthModule(oauth);
361
+
362
+ await warmPdsSecretLog(core);
363
+ const internalToken = writeInternalToken(home);
364
+ const app = server(core, uiECI, auth, oauth, identityService, internalToken);
365
+
366
+ if ((!port || !_.isInteger(port) || port < 1) && port !== 0) {
367
+ port = process.env.NODE_ENV === "test" ? 0 : 3000;
368
+ }
369
+ let httpServer: Server | undefined;
370
+ let listening = false;
371
+ try {
372
+ await new Promise<void>((resolve, reject) => {
373
+ httpServer = app.listen(port, () => {
374
+ listening = true;
375
+ if (httpServer) {
376
+ const addr = httpServer.address();
377
+ if (addr && typeof addr !== "string" && _.isInteger(addr.port)) {
378
+ // Get the actual port i.e. if they set port to 0 nodejs will assign you an available port
379
+ port = addr.port;
380
+ }
381
+ }
382
+ resolve(undefined);
383
+ });
384
+ // listen failures are an 'error' event. With no listener, Node throws.
385
+ httpServer.on("error", (err: NodeJS.ErrnoException) => {
386
+ if (!listening) {
387
+ reject(err);
388
+ return;
389
+ }
390
+ log.error("HTTP server error", { error: err });
391
+ });
392
+ });
393
+ } catch (err) {
394
+ const coded = err as NodeJS.ErrnoException;
395
+ if (coded.code === "EADDRINUSE") {
396
+ log.error(`Port ${port} is already in use`);
397
+ } else {
398
+ log.error("HTTP server failed to listen", { error: err, port });
399
+ }
400
+ if (httpServer) {
401
+ await new Promise<void>((resolve) => httpServer!.close(() => resolve()));
402
+ }
403
+ await rulesetLoaderHandle.close().catch(() => undefined);
404
+ await db.close().catch(() => undefined);
405
+ throw err;
406
+ }
407
+ if (typeof base_url !== "string") {
408
+ base_url = `http://localhost:${port}`;
409
+ } else if (
410
+ baseUrlFromCaller == null &&
411
+ (configuration.port === 0 || port !== settings.port)
412
+ ) {
413
+ base_url = `http://localhost:${port}`;
414
+ }
415
+ core.base_url = base_url;
416
+
417
+ await identityService.ensureWebvhForAllLoadedPicos();
418
+
419
+ log.info(`Listening at ${base_url}`);
420
+
421
+ if (uiECI) {
422
+ pf.event({
423
+ eci: uiECI,
424
+ domain: "engine",
425
+ name: "started",
426
+ data: { attrs: {} },
427
+ time: 0, // TODO remove this typescript requirement
428
+ }).catch((error) => {
429
+ log.error("Error signaling engine:started event", { error });
430
+ });
431
+ }
432
+
433
+ let shuttingDown = false;
434
+ async function shutdown() {
435
+ if (shuttingDown) {
436
+ return;
437
+ }
438
+ shuttingDown = true;
439
+ log.info("Shutting down pico-engine");
440
+ await new Promise<void>((resolve, reject) => {
441
+ if (!httpServer) {
442
+ resolve(undefined);
443
+ return;
444
+ }
445
+ httpServer.close((err) => (err ? reject(err) : resolve(undefined)));
446
+ });
447
+ await rulesetLoaderHandle.close();
448
+ await db.close();
449
+ log.info("Shutdown complete");
450
+ }
451
+
452
+ return {
453
+ version,
454
+
455
+ home,
456
+ port,
457
+ base_url,
458
+
459
+ pf,
460
+ core,
461
+ uiECI,
462
+ rsRegistry,
463
+ auth,
464
+ oauth,
465
+ identity,
466
+ identityService,
467
+ shutdown,
468
+ };
469
+ }