pico-engine 1.6.1 → 1.6.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,8 +5,10 @@ import * as makeDir from "make-dir";
5
5
  import * as path from "path";
6
6
  import { RulesetRegistryLoader } from "pico-engine-core";
7
7
  import { PicoDbKey, Ruleset } from "pico-framework";
8
+ import type { KrlLogger } from "krl-stdlib";
8
9
  import * as urlLib from "url";
9
10
  import { resolveBundledKrlUrl } from "./bundledKrl";
11
+ import { openClassicLevelWithRecovery } from "./levelDb";
10
12
  const krlCompiler = require("krl-compiler");
11
13
  const krlCompilerVersion = require("krl-compiler/package.json").version;
12
14
  const charwise = require("charwise");
@@ -34,15 +36,25 @@ interface DbUrlRuleset {
34
36
  };
35
37
  }
36
38
 
37
- export function RulesetRegistryLoaderFs(
38
- engineHomeDir: string
39
- ): RulesetRegistryLoader {
39
+ export interface RulesetRegistryLoaderFsHandle {
40
+ loader: RulesetRegistryLoader;
41
+ close: () => Promise<void>;
42
+ }
43
+
44
+ export async function createRulesetRegistryLoaderFs(
45
+ engineHomeDir: string,
46
+ log?: KrlLogger
47
+ ): Promise<RulesetRegistryLoaderFsHandle> {
40
48
  const rulesetDir = path.resolve(engineHomeDir, "rulesets");
41
49
 
42
- const db = new ClassicLevel<PicoDbKey, any>(path.resolve(engineHomeDir, "rulesets-db"), {
43
- keyEncoding: charwise,
44
- valueEncoding: safeJsonCodec,
45
- })
50
+ const db = await openClassicLevelWithRecovery<PicoDbKey, any>(
51
+ path.resolve(engineHomeDir, "rulesets-db"),
52
+ {
53
+ keyEncoding: charwise,
54
+ valueEncoding: safeJsonCodec,
55
+ },
56
+ log
57
+ );
46
58
 
47
59
  async function compileAndLoad(
48
60
  url: string,
@@ -91,7 +103,7 @@ export function RulesetRegistryLoaderFs(
91
103
  };
92
104
  }
93
105
 
94
- return {
106
+ const loader: RulesetRegistryLoader = {
95
107
  fetchKrl,
96
108
 
97
109
  async save({ ruleset, ...rest }) {
@@ -125,6 +137,11 @@ export function RulesetRegistryLoaderFs(
125
137
  };
126
138
  },
127
139
  };
140
+
141
+ return {
142
+ loader,
143
+ close: () => db.close(),
144
+ };
128
145
  }
129
146
 
130
147
  async function fetchKrl(url: string): Promise<string> {
package/src/bundledKrl.ts CHANGED
@@ -14,29 +14,167 @@ export const BUNDLED_KRL_FILES = [
14
14
  "io.picolabs.pds.krl",
15
15
  ];
16
16
 
17
+ /** Rulesets that only function when installed on the mesh root pico. */
18
+ export const ROOT_ONLY_RULESET_RIDS = new Set(["io.picolabs.oauth"]);
19
+
20
+ export interface EngineKrlSource {
21
+ rid: string;
22
+ filename: string;
23
+ relativePath: string;
24
+ url: string;
25
+ name?: string;
26
+ description?: string;
27
+ rootOnly?: boolean;
28
+ }
29
+
17
30
  export function bundledKrlDir(): string {
18
31
  return path.resolve(__dirname, "..", "krl");
19
32
  }
20
33
 
21
- export function resolveBundledKrlUrl(url: string): string {
22
- let filename: string | null = null;
23
- try {
24
- const parsed = new URL(url);
25
- if (parsed.protocol === "file:") {
26
- filename = path.basename(parsed.pathname);
34
+ function walkKrlFiles(dir: string, baseDir: string, out: string[]): void {
35
+ for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
36
+ const fullPath = path.join(dir, entry.name);
37
+ if (entry.isDirectory()) {
38
+ walkKrlFiles(fullPath, baseDir, out);
39
+ continue;
40
+ }
41
+ if (entry.isFile() && entry.name.endsWith(".krl")) {
42
+ out.push(path.relative(baseDir, fullPath).split(path.sep).join("/"));
27
43
  }
44
+ }
45
+ }
46
+
47
+ /** All `.krl` files under the engine's shipped `krl/` directory. */
48
+ export function listEngineKrlRelativePaths(): string[] {
49
+ const root = bundledKrlDir();
50
+ if (!fs.existsSync(root)) {
51
+ return [];
52
+ }
53
+ const paths: string[] = [];
54
+ walkKrlFiles(root, root, paths);
55
+ return paths.sort();
56
+ }
57
+
58
+ function ridFromRelativePath(relativePath: string): string {
59
+ const basename = path.basename(relativePath, ".krl");
60
+ return basename;
61
+ }
62
+
63
+ function readKrlMeta(
64
+ filePath: string
65
+ ): Pick<EngineKrlSource, "rid" | "name" | "description"> {
66
+ const text = fs.readFileSync(filePath, { encoding: "utf8", flag: "r" }).slice(0, 4000);
67
+ const ridMatch = text.match(/^ruleset\s+(\S+)/m);
68
+ const nameMatch = text.match(/\bname\s+"([^"]+)"/);
69
+ let description: string | undefined;
70
+ const descQuoted = text.match(/\bdescription\s+"([^"]+)"/);
71
+ if (descQuoted) {
72
+ description = descQuoted[1];
73
+ } else {
74
+ const descHere = text.match(/\bdescription\s+<<\s*([\s\S]*?)\s*>>/);
75
+ if (descHere) {
76
+ description = descHere[1].replace(/\s+/g, " ").trim();
77
+ }
78
+ }
79
+ return {
80
+ rid: ridMatch ? ridMatch[1] : ridFromRelativePath(path.basename(filePath)),
81
+ name: nameMatch ? nameMatch[1] : undefined,
82
+ description,
83
+ };
84
+ }
85
+
86
+ export function engineKrlHttpPath(relativePath: string): string {
87
+ return `/krl/${relativePath.split(path.sep).join("/")}`;
88
+ }
89
+
90
+ export function engineKrlFileUrl(
91
+ relativePath: string,
92
+ baseUrl?: string
93
+ ): string {
94
+ if (baseUrl) {
95
+ return `${baseUrl.replace(/\/$/, "")}${engineKrlHttpPath(relativePath)}`;
96
+ }
97
+ return toFileUrl(path.join(bundledKrlDir(), relativePath));
98
+ }
99
+
100
+ export function listEngineKrlSources(baseUrl?: string): EngineKrlSource[] {
101
+ const root = bundledKrlDir();
102
+ return listEngineKrlRelativePaths().map((relativePath) => {
103
+ const filePath = path.join(root, relativePath);
104
+ const meta = readKrlMeta(filePath);
105
+ const filename = path.basename(relativePath);
106
+ return {
107
+ rid: meta.rid,
108
+ filename,
109
+ relativePath,
110
+ url: engineKrlFileUrl(relativePath, baseUrl),
111
+ name: meta.name,
112
+ description: meta.description,
113
+ rootOnly: ROOT_ONLY_RULESET_RIDS.has(meta.rid),
114
+ };
115
+ });
116
+ }
117
+
118
+ /** Map a URL to a relative path under `krl/`, if it refers to an engine-shipped ruleset. */
119
+ export function engineKrlRelativeFromUrl(url: string): string | null {
120
+ let parsed: URL;
121
+ try {
122
+ parsed = new URL(url);
28
123
  } catch {
29
- return url;
124
+ return null;
30
125
  }
31
126
 
32
- if (!filename || !BUNDLED_KRL_FILES.includes(filename)) {
33
- return url;
127
+ if (parsed.protocol === "file:") {
128
+ const filePath = decodeURI(parsed.pathname).replace(/^\/([a-z]:\/)/i, "$1");
129
+ const normalized = path.normalize(filePath);
130
+ const root = bundledKrlDir();
131
+ if (normalized.startsWith(root + path.sep) || normalized === root) {
132
+ return path.relative(root, normalized).split(path.sep).join("/");
133
+ }
134
+ const basename = path.basename(normalized);
135
+ const candidate = path.join(root, basename);
136
+ if (basename.endsWith(".krl") && fs.existsSync(candidate)) {
137
+ return basename;
138
+ }
139
+ return null;
34
140
  }
35
141
 
36
- const localPath = path.join(bundledKrlDir(), filename);
37
- if (fs.existsSync(localPath)) {
38
- return toFileUrl(localPath);
142
+ if (parsed.protocol === "http:" || parsed.protocol === "https:") {
143
+ const match = parsed.pathname.match(/^\/krl\/(.+\.krl)$/);
144
+ if (match) {
145
+ const relativePath = match[1].split("/").join(path.sep);
146
+ const candidate = path.join(bundledKrlDir(), relativePath);
147
+ if (fs.existsSync(candidate)) {
148
+ return match[1];
149
+ }
150
+ }
39
151
  }
40
152
 
153
+ return null;
154
+ }
155
+
156
+ export function resolveEngineKrlFile(relativePath: string): string | null {
157
+ const normalized = relativePath.split("/").join(path.sep);
158
+ const candidate = path.resolve(bundledKrlDir(), normalized);
159
+ const root = bundledKrlDir();
160
+ if (
161
+ candidate.startsWith(root + path.sep) &&
162
+ fs.existsSync(candidate) &&
163
+ candidate.endsWith(".krl")
164
+ ) {
165
+ return candidate;
166
+ }
167
+ return null;
168
+ }
169
+
170
+ export function resolveBundledKrlUrl(url: string): string {
171
+ const relativePath = engineKrlRelativeFromUrl(url);
172
+ if (!relativePath) {
173
+ return url;
174
+ }
175
+ const localPath = resolveEngineKrlFile(relativePath);
176
+ if (localPath) {
177
+ return toFileUrl(localPath);
178
+ }
41
179
  return url;
42
180
  }
package/src/cli.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { PicoEngineConfiguration, startEngine } from "./index";
1
+ import { PicoEngine, PicoEngineConfiguration, startEngine } from "./index";
2
2
  const version = require("../package.json").version;
3
3
 
4
4
  const args: {
@@ -64,8 +64,35 @@ if (allowSelfSignupEnv === "true" || allowSelfSignupEnv === "1") {
64
64
  env.allowSelfSignup = false;
65
65
  }
66
66
 
67
- startEngine(env).catch((err) => {
68
- console.error("Failed to start engine.");
69
- console.error(err);
70
- process.exit(1);
71
- });
67
+ function installGracefulShutdown(engine: PicoEngine) {
68
+ let shuttingDown = false;
69
+
70
+ const onSignal = (signal: NodeJS.Signals) => {
71
+ if (shuttingDown) {
72
+ return;
73
+ }
74
+ shuttingDown = true;
75
+ console.log(`Received ${signal}; shutting down`);
76
+ engine
77
+ .shutdown()
78
+ .then(() => process.exit(0))
79
+ .catch((err) => {
80
+ console.error("Shutdown failed.");
81
+ console.error(err);
82
+ process.exit(1);
83
+ });
84
+ };
85
+
86
+ process.on("SIGTERM", onSignal);
87
+ process.on("SIGINT", onSignal);
88
+ }
89
+
90
+ startEngine(env)
91
+ .then((engine) => {
92
+ installGracefulShutdown(engine);
93
+ })
94
+ .catch((err) => {
95
+ console.error("Failed to start engine.");
96
+ console.error(err);
97
+ process.exit(1);
98
+ });
@@ -35,7 +35,7 @@ export interface CrossPicoDeps {
35
35
  }
36
36
 
37
37
  function remotePeerDid(sub: PeerSubscriptionRecord): string {
38
- const did = sub.remotePeerDid || sub.remoteDid;
38
+ const did = sub.remotePeerDid || sub.remoteDid || sub.remoteWebvhDid;
39
39
  if (!did) {
40
40
  throw new IdentityError("Subscription missing remote peer DID", 400);
41
41
  }
@@ -114,20 +114,37 @@ export async function crossPicoEvent(
114
114
  to &&
115
115
  (await usesLocalDispatch(deps.store, deps.pf, fromPicoId, to, sub))
116
116
  ) {
117
- const recipientSub = await findRecipientSub(deps.store, to, subscriptionId);
118
- return deps.pf.eventWait(
119
- {
120
- eci: recipientSub.rxEci!,
121
- domain: event.domain,
122
- name: event.name,
123
- data: { attrs },
124
- time: 0,
125
- },
126
- fromPicoId
127
- );
117
+ const rxEci = await resolveRecipientRxEci(deps, to, subscriptionId);
118
+ if (rxEci) {
119
+ return deps.pf.eventWait(
120
+ {
121
+ eci: rxEci,
122
+ domain: event.domain,
123
+ name: event.name,
124
+ data: { attrs },
125
+ time: 0,
126
+ },
127
+ fromPicoId
128
+ );
129
+ }
128
130
  }
129
131
 
130
- return sendSkyEvent(deps, fromPicoId, sub, event, attrs);
132
+ try {
133
+ return await sendSkyEvent(deps, fromPicoId, sub, event, attrs);
134
+ } catch (err) {
135
+ if (isRemovalNotification(event.name)) {
136
+ return { eid: "none", responses: [] };
137
+ }
138
+ throw err;
139
+ }
140
+ }
141
+
142
+ function isRemovalNotification(name: string): boolean {
143
+ return (
144
+ name === "inbound_removal" ||
145
+ name === "outbound_removal" ||
146
+ name === "established_removal"
147
+ );
131
148
  }
132
149
 
133
150
  export async function crossPicoQuery(
@@ -150,34 +167,88 @@ export async function crossPicoQuery(
150
167
  to &&
151
168
  (await usesLocalDispatch(deps.store, deps.pf, fromPicoId, to, sub))
152
169
  ) {
153
- const recipientSub = await findRecipientSub(deps.store, to, subscriptionId);
154
- return deps.pf.query(
155
- {
156
- eci: recipientSub.rxEci!,
157
- rid: query.rid,
158
- name: query.name,
159
- args: query.args || {},
160
- },
161
- fromPicoId
162
- );
170
+ const rxEci = await resolveRecipientRxEci(deps, to, subscriptionId);
171
+ if (rxEci) {
172
+ return deps.pf.query(
173
+ {
174
+ eci: rxEci,
175
+ rid: query.rid,
176
+ name: query.name,
177
+ args: query.args || {},
178
+ },
179
+ fromPicoId
180
+ );
181
+ }
163
182
  }
164
183
 
165
184
  return sendSkyQuery(deps, fromPicoId, sub, query);
166
185
  }
167
186
 
168
- async function findRecipientSub(
169
- store: IdentityStore,
187
+ function deliveryEci(
188
+ pf: PicoFramework,
189
+ picoId: string,
190
+ preferred?: string
191
+ ): string {
192
+ if (preferred) {
193
+ return preferred;
194
+ }
195
+ const pico = pf.loadedPicos().find((p) => p.id === picoId);
196
+ if (!pico) {
197
+ throw new IdentityError(`Pico ${picoId} not loaded`, 404);
198
+ }
199
+ for (const ch of Object.values(pico.channels)) {
200
+ const ro = ch.toReadOnly();
201
+ if (ro.tags.includes("wellKnown_Rx")) {
202
+ return ro.id;
203
+ }
204
+ }
205
+ for (const ch of Object.values(pico.channels)) {
206
+ const ro = ch.toReadOnly();
207
+ if (ro.tags.includes("allow-all")) {
208
+ return ro.id;
209
+ }
210
+ }
211
+ const first = Object.keys(pico.channels)[0];
212
+ if (!first) {
213
+ throw new IdentityError("Pico has no channel for cross-pico delivery", 404);
214
+ }
215
+ return first;
216
+ }
217
+
218
+ /** Resolve Rx ECI on the recipient for local dispatch (identity store, then inbound bus). */
219
+ async function resolveRecipientRxEci(
220
+ deps: CrossPicoDeps,
170
221
  toPicoId: string,
171
222
  subscriptionId: string
172
- ): Promise<PeerSubscriptionRecord> {
173
- const recipientSub = await store.getPeerSubscription(toPicoId, subscriptionId);
174
- if (!recipientSub?.rxEci) {
175
- throw new IdentityError(
176
- `Recipient subscription ${subscriptionId} missing rxEci`,
177
- 404
178
- );
223
+ ): Promise<string | undefined> {
224
+ const recipientSub = await deps.store.getPeerSubscription(
225
+ toPicoId,
226
+ subscriptionId
227
+ );
228
+ if (recipientSub?.rxEci) {
229
+ return recipientSub.rxEci;
179
230
  }
180
- return recipientSub;
231
+
232
+ try {
233
+ const queryEci = deliveryEci(deps.pf, toPicoId, recipientSub?.rxEci);
234
+ const inbound = (await deps.pf.query(
235
+ {
236
+ eci: queryEci,
237
+ rid: "io.picolabs.subscription",
238
+ name: "inbound",
239
+ args: {},
240
+ },
241
+ toPicoId
242
+ )) as Array<{ Id?: string; Rx?: string }>;
243
+ const bus = inbound.find((b) => b.Id === subscriptionId);
244
+ if (typeof bus?.Rx === "string" && bus.Rx.length > 0) {
245
+ return bus.Rx;
246
+ }
247
+ } catch {
248
+ // fall through to DIDComm or no-op
249
+ }
250
+
251
+ return undefined;
181
252
  }
182
253
 
183
254
  async function sendSkyEvent(
@@ -101,6 +101,11 @@ export async function sendSkyIntro(
101
101
  picoId: string,
102
102
  input: SkyIntroSendInput
103
103
  ): Promise<{ messageId: string; peerDid: string }> {
104
+ const senderWebvh = await deps.store.getWebvhDid(picoId);
105
+ if (senderWebvh && input.targetDid === senderWebvh) {
106
+ throw new IdentityError("Cannot form a relationship with self", 400);
107
+ }
108
+
104
109
  const established = await establishSubscriptionIdentity(
105
110
  deps.store,
106
111
  deps.pf,
@@ -126,7 +131,6 @@ export async function sendSkyIntro(
126
131
  subscription_id: input.subscriptionId,
127
132
  };
128
133
 
129
- const senderWebvh = await deps.store.getWebvhDid(picoId);
130
134
  if (senderWebvh) {
131
135
  body.sender_webvh = senderWebvh;
132
136
  try {
package/src/index.ts CHANGED
@@ -1,17 +1,22 @@
1
1
  import { krl, KrlLogger, makeKrlLogger } from "krl-stdlib";
2
- import { ClassicLevel } from "classic-level";
3
2
  import * as _ from "lodash";
4
3
  import * as makeDir from "make-dir";
5
4
  import * as path from "path";
5
+ import type { Server } from "http";
6
6
  import { PicoEngineCore, RulesetRegistry } from "pico-engine-core";
7
7
  import { PicoDbKey, PicoFramework } from "pico-framework";
8
8
  import { AuthService, WebAuthnAdapter, makeWebAuthnAdapter } from "./auth";
9
9
  import { IdentityStore, IdentityService, initDidoModule } from "./identity";
10
10
  import { registerWebvhRoutes } from "./identity/webvhRoutes";
11
11
  import { OAuthService, initOAuthModule } from "./oauth";
12
- import { getPicoLogs, makeRotatingFileLogWriter } from "./logging";
12
+ import {
13
+ getPicoLogs,
14
+ makeEngineLogWriter,
15
+ resolveEngineLogFilePath,
16
+ } from "./logging";
17
+ import { openClassicLevelWithRecovery } from "./levelDb";
13
18
  import { provisionRoot, uiECIForRoot } from "./provisionRoot";
14
- import { RulesetRegistryLoaderFs } from "./RulesetRegistryLoaderFs";
19
+ import { createRulesetRegistryLoaderFs } from "./RulesetRegistryLoaderFs";
15
20
  import { server } from "./server";
16
21
  const charwise = require("charwise");
17
22
  const safeJsonCodec = require("level-json-coerce-null");
@@ -111,6 +116,9 @@ export interface PicoEngine {
111
116
  oauth: OAuthService;
112
117
  identity: IdentityStore;
113
118
  identityService: IdentityService;
119
+
120
+ /** Stop HTTP and close LevelDB stores (for container SIGTERM). */
121
+ shutdown(): Promise<void>;
114
122
  }
115
123
 
116
124
  export async function startEngine(
@@ -134,22 +142,32 @@ export async function startEngine(
134
142
  }
135
143
  await makeDir(home);
136
144
 
137
- const logFilePath = path.resolve(home, "pico-engine.log");
145
+ const logFilePath = resolveEngineLogFilePath(home);
138
146
  const log = configuration.log
139
147
  ? configuration.log
140
- : makeKrlLogger(makeRotatingFileLogWriter(logFilePath));
148
+ : makeKrlLogger(makeEngineLogWriter(logFilePath));
141
149
 
142
- const core = new PicoEngineCore({
143
- db: new ClassicLevel<PicoDbKey, any>(path.resolve(home, "db"), {
150
+ const db = await openClassicLevelWithRecovery<PicoDbKey, any>(
151
+ path.resolve(home, "db"),
152
+ {
144
153
  keyEncoding: charwise,
145
154
  valueEncoding: safeJsonCodec,
146
- }),
147
- rsRegLoader: RulesetRegistryLoaderFs(home),
155
+ },
156
+ log
157
+ );
158
+ const rulesetLoaderHandle = await createRulesetRegistryLoaderFs(home, log);
159
+
160
+ const core = new PicoEngineCore({
161
+ db,
162
+ rsRegLoader: rulesetLoaderHandle.loader,
148
163
  log,
149
164
  modules: configuration.modules,
150
165
  useEventInputTime: configuration.useEventInputTime,
151
166
  autoCreateRootPico: configuration.autoCreateRootPico ?? false,
152
167
  getPicoLogs(picoId) {
168
+ if (!logFilePath) {
169
+ return Promise.resolve([]);
170
+ }
153
171
  return getPicoLogs(logFilePath, picoId);
154
172
  },
155
173
  });
@@ -168,7 +186,14 @@ export async function startEngine(
168
186
  };
169
187
  core.modules["dido"] = initDidoModule(identityService);
170
188
 
171
- await core.start();
189
+ try {
190
+ await core.start();
191
+ } catch (err) {
192
+ log.error("Engine startup failed", { error: err });
193
+ await rulesetLoaderHandle.close().catch(() => undefined);
194
+ await db.close().catch(() => undefined);
195
+ throw err;
196
+ }
172
197
  const rsRegistry = core.rsRegistry;
173
198
 
174
199
  // Fresh engines boot with zero roots. Adopt + (re)provision only when roots
@@ -233,10 +258,11 @@ export async function startEngine(
233
258
  if ((!port || !_.isInteger(port) || port < 1) && port !== 0) {
234
259
  port = process.env.NODE_ENV === "test" ? 0 : 3000;
235
260
  }
236
- await new Promise((resolve) => {
237
- const listener = app.listen(port, () => {
238
- if (listener) {
239
- const addr = listener.address();
261
+ let httpServer: Server | undefined;
262
+ await new Promise<void>((resolve) => {
263
+ httpServer = app.listen(port, () => {
264
+ if (httpServer) {
265
+ const addr = httpServer.address();
240
266
  if (addr && typeof addr !== "string" && _.isInteger(addr.port)) {
241
267
  // Get the actual port i.e. if they set port to 0 nodejs will assign you an available port
242
268
  port = addr.port;
@@ -266,6 +292,25 @@ export async function startEngine(
266
292
  });
267
293
  }
268
294
 
295
+ let shuttingDown = false;
296
+ async function shutdown() {
297
+ if (shuttingDown) {
298
+ return;
299
+ }
300
+ shuttingDown = true;
301
+ log.info("Shutting down pico-engine");
302
+ await new Promise<void>((resolve, reject) => {
303
+ if (!httpServer) {
304
+ resolve(undefined);
305
+ return;
306
+ }
307
+ httpServer.close((err) => (err ? reject(err) : resolve(undefined)));
308
+ });
309
+ await rulesetLoaderHandle.close();
310
+ await db.close();
311
+ log.info("Shutdown complete");
312
+ }
313
+
269
314
  return {
270
315
  version,
271
316
 
@@ -280,5 +325,6 @@ export async function startEngine(
280
325
  oauth,
281
326
  identity,
282
327
  identityService,
328
+ shutdown,
283
329
  };
284
330
  }