pico-engine 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,16 +1,14 @@
1
1
  import fetch from "cross-fetch";
2
2
  import * as fs from "fs";
3
- import leveldown from "leveldown";
4
- import { default as level } from "levelup";
3
+ import { ClassicLevel } from "classic-level";
5
4
  import * as makeDir from "make-dir";
6
5
  import * as path from "path";
7
6
  import { RulesetRegistryLoader } from "pico-engine-core";
8
- import { Ruleset } from "pico-framework";
7
+ import { PicoDbKey, Ruleset } from "pico-framework";
9
8
  import * as urlLib from "url";
10
9
  const krlCompiler = require("krl-compiler");
11
10
  const krlCompilerVersion = require("krl-compiler/package.json").version;
12
11
  const charwise = require("charwise");
13
- const encode = require("encoding-down");
14
12
  const safeJsonCodec = require("level-json-coerce-null");
15
13
 
16
14
  function getUrlFilename(url: string): string | null {
@@ -40,12 +38,10 @@ export function RulesetRegistryLoaderFs(
40
38
  ): RulesetRegistryLoader {
41
39
  const rulesetDir = path.resolve(engineHomeDir, "rulesets");
42
40
 
43
- const db = level(
44
- encode(leveldown(path.resolve(engineHomeDir, "rulesets-db")), {
45
- keyEncoding: charwise,
46
- valueEncoding: safeJsonCodec,
47
- })
48
- );
41
+ const db = new ClassicLevel<PicoDbKey, any>(path.resolve(engineHomeDir, "rulesets-db"), {
42
+ keyEncoding: charwise,
43
+ valueEncoding: safeJsonCodec,
44
+ })
49
45
 
50
46
  async function compileAndLoad(
51
47
  url: string,
package/src/cli.ts CHANGED
@@ -23,7 +23,7 @@ USAGE
23
23
 
24
24
  Environment Variables:
25
25
 
26
- PORT - The port the http server should listen on. By default it's 8080
26
+ PORT - The port the http server should listen on. By default it's 3000
27
27
  PICO_ENGINE_HOME - Where the database and other files should be stored. By default it's ~/.pico-engine/
28
28
  PICO_ENGINE_BASE_URL - The public url prefix to reach this engine. By default it's http://localhost:$PORT
29
29
  `);
package/src/index.ts CHANGED
@@ -1,187 +1,192 @@
1
- import { krl, KrlLogger, makeKrlLogger } from "krl-stdlib";
2
- import leveldown from "leveldown";
3
- import * as _ from "lodash";
4
- import * as makeDir from "make-dir";
5
- import * as path from "path";
6
- import { PicoEngineCore, RulesetRegistry } from "pico-engine-core";
7
- import { PicoFramework } from "pico-framework";
8
- import { getPicoLogs, makeRotatingFileLogWriter } from "./logging";
9
- import { RulesetRegistryLoaderFs } from "./RulesetRegistryLoaderFs";
10
- import { server } from "./server";
11
- import { toFileUrl } from "./utils/toFileUrl";
12
- import * as fs from "fs";
13
- import { reject } from "lodash";
14
-
15
- const homeDir = require("home-dir");
16
- const version = require("../package.json").version;
17
-
18
- /**
19
- * Configuration options that may be set by the user
20
- */
21
- export interface PicoEngineConfiguration {
22
- /**
23
- * The absolute path to the folder where the engine should store the database and logs.
24
- *
25
- * Default: "~/.pico-engine/"
26
- */
27
- home?: string;
28
-
29
- /**
30
- * The port number the http server should listen on.
31
- *
32
- * If you want an available port assigned to you, set it to 0
33
- *
34
- * Default: 3000
35
- */
36
- port?: number;
37
-
38
- /**
39
- * The base url others should use when addressing your engine.
40
- *
41
- * Default: "http://localhost:3000"
42
- */
43
- base_url?: string;
44
-
45
- /**
46
- * Provide any custom krl modules
47
- */
48
- modules?: { [domain: string]: krl.Module };
49
-
50
- /**
51
- * Trust event.time input. Used for testing
52
- */
53
- useEventInputTime?: boolean;
54
-
55
- log?: KrlLogger;
56
- }
57
-
58
- export interface PicoEngine {
59
- version: string;
60
-
61
- home: string;
62
- port: number;
63
- base_url: string;
64
-
65
- pf: PicoFramework;
66
- uiECI: string;
67
- rsRegistry: RulesetRegistry;
68
- }
69
-
70
- export async function startEngine(
71
- configuration: PicoEngineConfiguration = {}
72
- ): Promise<PicoEngine> {
73
- let home = configuration.home;
74
- let port = configuration.port;
75
- let base_url = configuration.base_url;
76
-
77
- if (typeof home !== "string") {
78
- home = homeDir(".pico-engine") as string;
79
- }
80
- await makeDir(home);
81
-
82
- const logFilePath = path.resolve(home, "pico-engine.log");
83
- const log = configuration.log
84
- ? configuration.log
85
- : makeKrlLogger(makeRotatingFileLogWriter(logFilePath));
86
-
87
- const core = new PicoEngineCore({
88
- leveldown: leveldown(path.resolve(home, "db")) as any,
89
- rsRegLoader: RulesetRegistryLoaderFs(home),
90
- log,
91
- modules: configuration.modules,
92
- useEventInputTime: configuration.useEventInputTime,
93
- getPicoLogs(picoId) {
94
- return getPicoLogs(logFilePath, picoId);
95
- },
96
- });
97
- await core.start();
98
- const rsRegistry = core.rsRegistry;
99
- const pf = core.picoFramework;
100
-
101
- const krl_urls: string[] = [
102
- toFileUrl(
103
- path.resolve(__dirname, "..", "krl", "io.picolabs.pico-engine-ui.krl")
104
- ),
105
- toFileUrl(path.resolve(__dirname, "..", "krl", "io.picolabs.wrangler.krl")),
106
- toFileUrl(
107
- path.resolve(__dirname, "..", "krl", "io.picolabs.subscription.krl")
108
- ),
109
- ];
110
- for (const url of krl_urls) {
111
- const { ruleset } = await rsRegistry.flush(url);
112
- await pf.rootPico.install(ruleset, { url, config: {} });
113
- }
114
-
115
- let uiChannel = pf.rootPico
116
- .toReadOnly()
117
- .channels.find(
118
- (chann) => "engine,ui" === chann.tags.slice(0).sort().join(",")
119
- );
120
- if (!uiChannel) {
121
- uiChannel = (
122
- await pf.rootPico.newChannel({
123
- tags: ["engine", "ui"],
124
- eventPolicy: {
125
- allow: [{ domain: "engine_ui", name: "setup" }],
126
- deny: [],
127
- },
128
- })
129
- ).toReadOnly();
130
- }
131
- await pf.eventWait({
132
- eci: uiChannel.id,
133
- domain: "engine_ui",
134
- name: "setup",
135
- data: { attrs: {} },
136
- time: 0,
137
- });
138
-
139
- const uiECI = uiChannel.id;
140
-
141
- const app = server(core, uiECI);
142
-
143
- if ((!port || !_.isInteger(port) || port < 1) && port !== 0) {
144
- port = 3000;
145
- }
146
- await new Promise((resolve) => {
147
- const listener = app.listen(port, () => {
148
- if (listener) {
149
- const addr = listener.address();
150
- if (addr && typeof addr !== "string" && _.isInteger(addr.port)) {
151
- // Get the actual port i.e. if they set port to 0 nodejs will assign you an available port
152
- port = addr.port;
153
- }
154
- }
155
- resolve(undefined);
156
- });
157
- });
158
- if (typeof base_url !== "string") {
159
- base_url = `http://localhost:${port}`;
160
- }
161
- core.base_url = base_url;
162
-
163
- log.info(`Listening at ${base_url}`);
164
-
165
- pf.event({
166
- eci: uiECI,
167
- domain: "engine",
168
- name: "started",
169
- data: { attrs: {} },
170
- time: 0, // TODO remove this typescript requirement
171
- }).catch((error) => {
172
- log.error("Error signaling engine:started event", { error });
173
- // TODO signal all errors engine:error
174
- });
175
-
176
- return {
177
- version,
178
-
179
- home,
180
- port,
181
- base_url,
182
-
183
- pf,
184
- uiECI,
185
- rsRegistry,
186
- };
187
- }
1
+ import { krl, KrlLogger, makeKrlLogger } from "krl-stdlib";
2
+ import { ClassicLevel } from "classic-level";
3
+ import * as _ from "lodash";
4
+ import * as makeDir from "make-dir";
5
+ import * as path from "path";
6
+ import { PicoEngineCore, RulesetRegistry } from "pico-engine-core";
7
+ import { PicoDbKey, PicoFramework } from "pico-framework";
8
+ import { getPicoLogs, makeRotatingFileLogWriter } from "./logging";
9
+ import { RulesetRegistryLoaderFs } from "./RulesetRegistryLoaderFs";
10
+ import { server } from "./server";
11
+ import { toFileUrl } from "./utils/toFileUrl";
12
+ const charwise = require("charwise");
13
+ const safeJsonCodec = require("level-json-coerce-null");
14
+
15
+ const homeDir = require("home-dir");
16
+ const version = require("../package.json").version;
17
+
18
+ /**
19
+ * Configuration options that may be set by the user
20
+ */
21
+ export interface PicoEngineConfiguration {
22
+ /**
23
+ * The absolute path to the folder where the engine should store the database and logs.
24
+ *
25
+ * Default: "~/.pico-engine/"
26
+ */
27
+ home?: string;
28
+
29
+ /**
30
+ * The port number the http server should listen on.
31
+ *
32
+ * If you want an available port assigned to you, set it to 0
33
+ *
34
+ * Default: 3000
35
+ */
36
+ port?: number;
37
+
38
+ /**
39
+ * The base url others should use when addressing your engine.
40
+ *
41
+ * Default: "http://localhost:3000"
42
+ */
43
+ base_url?: string;
44
+
45
+ /**
46
+ * Provide any custom krl modules
47
+ */
48
+ modules?: { [domain: string]: krl.Module };
49
+
50
+ /**
51
+ * Trust event.time input. Used for testing
52
+ */
53
+ useEventInputTime?: boolean;
54
+
55
+ log?: KrlLogger;
56
+ }
57
+
58
+ export interface PicoEngine {
59
+ version: string;
60
+
61
+ home: string;
62
+ port: number;
63
+ base_url: string;
64
+
65
+ pf: PicoFramework;
66
+ uiECI: string;
67
+ rsRegistry: RulesetRegistry;
68
+ }
69
+
70
+ export async function startEngine(
71
+ configuration: PicoEngineConfiguration = {}
72
+ ): Promise<PicoEngine> {
73
+ let home = configuration.home;
74
+ let port = configuration.port;
75
+ let base_url = configuration.base_url;
76
+
77
+ if (typeof home !== "string") {
78
+ home = homeDir(".pico-engine") as string;
79
+ }
80
+ await makeDir(home);
81
+
82
+ const logFilePath = path.resolve(home, "pico-engine.log");
83
+ const log = configuration.log
84
+ ? configuration.log
85
+ : makeKrlLogger(makeRotatingFileLogWriter(logFilePath));
86
+
87
+ const core = new PicoEngineCore({
88
+ db: new ClassicLevel<PicoDbKey, any>(path.resolve(home, "db"), {
89
+ keyEncoding: charwise,
90
+ valueEncoding: safeJsonCodec,
91
+ }),
92
+ rsRegLoader: RulesetRegistryLoaderFs(home),
93
+ log,
94
+ modules: configuration.modules,
95
+ useEventInputTime: configuration.useEventInputTime,
96
+ getPicoLogs(picoId) {
97
+ return getPicoLogs(logFilePath, picoId);
98
+ },
99
+ });
100
+ await core.start();
101
+ const rsRegistry = core.rsRegistry;
102
+ const pf = core.picoFramework;
103
+
104
+ const krl_urls: string[] = [
105
+ toFileUrl(
106
+ path.resolve(__dirname, "..", "krl", "io.picolabs.pico-engine-ui.krl")
107
+ ),
108
+ toFileUrl(path.resolve(__dirname, "..", "krl", "io.picolabs.wrangler.krl")),
109
+ toFileUrl(
110
+ path.resolve(__dirname, "..", "krl", "io.picolabs.subscription.krl")
111
+ ),
112
+ toFileUrl(path.resolve(__dirname, "..", "krl", "io.picolabs.did-o.krl")),
113
+ toFileUrl(path.resolve(__dirname, "..", "krl", "io.picolabs.pds.krl")),
114
+ ];
115
+ for (const url of krl_urls) {
116
+ const { ruleset } = await rsRegistry.flush(url);
117
+ await pf.rootPico.install(ruleset, { url, config: {} });
118
+ }
119
+
120
+ let uiChannel = pf.rootPico
121
+ .toReadOnly()
122
+ .channels.find(
123
+ (chann) => "engine,ui" === chann.tags.slice(0).sort().join(",")
124
+ );
125
+ if (!uiChannel) {
126
+ uiChannel = (
127
+ await pf.rootPico.newChannel({
128
+ tags: ["engine", "ui"],
129
+ eventPolicy: {
130
+ allow: [{ domain: "engine_ui", name: "setup" }],
131
+ deny: [],
132
+ },
133
+ })
134
+ ).toReadOnly();
135
+ }
136
+ await pf.eventWait({
137
+ eci: uiChannel.id,
138
+ domain: "engine_ui",
139
+ name: "setup",
140
+ data: { attrs: {} },
141
+ time: 0,
142
+ });
143
+
144
+ const uiECI = uiChannel.id;
145
+
146
+ const app = server(core, uiECI);
147
+
148
+ if ((!port || !_.isInteger(port) || port < 1) && port !== 0) {
149
+ port = 3000;
150
+ }
151
+ await new Promise((resolve) => {
152
+ const listener = app.listen(port, () => {
153
+ if (listener) {
154
+ const addr = listener.address();
155
+ if (addr && typeof addr !== "string" && _.isInteger(addr.port)) {
156
+ // Get the actual port i.e. if they set port to 0 nodejs will assign you an available port
157
+ port = addr.port;
158
+ }
159
+ }
160
+ resolve(undefined);
161
+ });
162
+ });
163
+ if (typeof base_url !== "string") {
164
+ base_url = `http://localhost:${port}`;
165
+ }
166
+ core.base_url = base_url;
167
+
168
+ log.info(`Listening at ${base_url}`);
169
+
170
+ pf.event({
171
+ eci: uiECI,
172
+ domain: "engine",
173
+ name: "started",
174
+ data: { attrs: {} },
175
+ time: 0, // TODO remove this typescript requirement
176
+ }).catch((error) => {
177
+ log.error("Error signaling engine:started event", { error });
178
+ // TODO signal all errors engine:error
179
+ });
180
+
181
+ return {
182
+ version,
183
+
184
+ home,
185
+ port,
186
+ base_url,
187
+
188
+ pf,
189
+ uiECI,
190
+ rsRegistry,
191
+ };
192
+ }
package/src/logging.ts CHANGED
@@ -1,8 +1,7 @@
1
1
  import * as fs from "fs";
2
+ import * as readline from "readline";
2
3
  import { krlLogLevelCodeToHuman, PicoLogEntry } from "krl-stdlib";
3
4
  import * as path from "path";
4
- const split = require("split");
5
- const through2 = require("through2");
6
5
 
7
6
  const rfs = require("rotating-file-stream");
8
7
 
@@ -21,14 +20,14 @@ function getRotatingFileStream(filePath: string): NodeJS.WritableStream {
21
20
 
22
21
  size: "100M", // rotate every 10 MegaBytes written
23
22
  maxFiles: 12,
24
- }
23
+ },
25
24
  ) as NodeJS.WritableStream;
26
25
  }
27
26
  return logStreams[filePath];
28
27
  }
29
28
 
30
29
  export function makeRotatingFileLogWriter(
31
- filePath: string
30
+ filePath: string,
32
31
  ): (line: string) => void {
33
32
  const fileStream = getRotatingFileStream(filePath);
34
33
  const isTest = process.env.NODE_ENV === "test";
@@ -45,50 +44,40 @@ export function makeRotatingFileLogWriter(
45
44
 
46
45
  export async function getPicoLogs(
47
46
  filePath: string,
48
- picoId: string
47
+ picoId: string,
49
48
  ): Promise<PicoLogEntry[]> {
50
- return new Promise((resolve, reject) => {
51
- const output: PicoLogEntry[] = [];
52
- fs.createReadStream(filePath)
53
- .pipe(split())
54
- .pipe(
55
- through2((chunk: any, enc: any, next: any) => {
56
- const line = chunk.toString();
57
- if (line.indexOf(picoId) < 0) {
58
- // not my pico
59
- return next();
60
- }
61
- let entry;
62
- try {
63
- entry = JSON.parse(line);
64
- } catch (err) {}
65
- if (!entry || entry.picoId !== picoId) {
66
- // not my pico
67
- return next();
68
- }
69
- const time = new Date(entry.time);
70
- if (Date.now() - time.getTime() > 1000 * 60 * 60 * 12) {
71
- // too old
72
- return next();
73
- }
74
-
75
- const out: PicoLogEntry = {
76
- ...entry,
77
- level: krlLogLevelCodeToHuman[entry.level] || `${entry.level}`,
78
- time: entry.time,
79
- txnId: entry.txnId,
80
- };
81
- delete (out as any).picoId;
82
- output.push(out);
49
+ const output: PicoLogEntry[] = [];
83
50
 
84
- next();
85
- })
86
- )
87
- .on("finish", () => {
88
- resolve(output);
89
- })
90
- .on("error", (err: any) => {
91
- reject(err);
92
- });
51
+ const rl = readline.createInterface({
52
+ input: fs.createReadStream(filePath),
93
53
  });
54
+
55
+ for await (const line of rl) {
56
+ // Each line in the readline input will be successively available here as
57
+ // `line`.
58
+ if (line.indexOf(picoId) < 0) {
59
+ continue; // not my pico
60
+ }
61
+ let entry;
62
+ try {
63
+ entry = JSON.parse(line);
64
+ } catch (err) {}
65
+ if (!entry || entry.picoId !== picoId) {
66
+ continue; // not my pico
67
+ }
68
+ const time = new Date(entry.time);
69
+ if (Date.now() - time.getTime() > 1000 * 60 * 60 * 12) {
70
+ continue; // too old
71
+ }
72
+
73
+ const out: PicoLogEntry = {
74
+ ...entry,
75
+ level: krlLogLevelCodeToHuman[entry.level] || `${entry.level}`,
76
+ time: entry.time,
77
+ txnId: entry.txnId,
78
+ };
79
+ delete (out as any).picoId;
80
+ output.push(out);
81
+ }
82
+ return output;
94
83
  }
package/src/server.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  import * as bodyParser from "body-parser";
2
2
  import * as express from "express";
3
3
  import { Express, Request } from "express";
4
- import * as helmet from "helmet";
4
+ import helmet from "helmet";
5
5
  import * as _ from "lodash";
6
6
  import * as path from "path";
7
7
  import { PicoEngineCore } from "pico-engine-core";
@@ -31,7 +31,18 @@ export function server(core: PicoEngineCore, uiECI: string): Express {
31
31
  next();
32
32
  }
33
33
  });
34
- app.use(helmet());
34
+ // Helmet 7+ CSP defaults include upgrade-insecure-requests, which breaks the
35
+ // UI on plain HTTP (browser upgrades /api/* to https and TLS fails).
36
+ app.use(
37
+ helmet({
38
+ contentSecurityPolicy: {
39
+ directives: {
40
+ ...helmet.contentSecurityPolicy.getDefaultDirectives(),
41
+ "upgrade-insecure-requests": null,
42
+ },
43
+ },
44
+ })
45
+ );
35
46
  app.use(express.static(path.resolve(__dirname, "..", "public")));
36
47
  app.use(
37
48
  bodyParser.json({