rudderstash 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -1
- package/dist/rudderstash.js +420 -10
- package/package.json +1 -1
package/README.md
CHANGED
package/dist/rudderstash.js
CHANGED
|
@@ -1,11 +1,421 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
2
|
+
|
|
3
|
+
// rudderstash.ts
|
|
4
|
+
import { writeFile as writeFile2, readdir as readdir2, readFile as readFile2 } from "fs/promises";
|
|
5
|
+
import { fileURLToPath } from "url";
|
|
6
|
+
import { dirname, join } from "path";
|
|
7
|
+
import { register } from "node:module";
|
|
8
|
+
import "colors";
|
|
9
|
+
import dotenv from "dotenv";
|
|
10
|
+
import yargs from "yargs";
|
|
11
|
+
import { hideBin } from "yargs/helpers";
|
|
12
|
+
import { diffJson } from "diff";
|
|
13
|
+
|
|
14
|
+
// rudderstash.lib.ts
|
|
15
|
+
import { existsSync } from "fs";
|
|
16
|
+
import { writeFile, readdir, readFile } from "fs/promises";
|
|
17
|
+
import { createHash } from "crypto";
|
|
18
|
+
var VERSION = "0.1.5";
|
|
19
|
+
var BUILD_INFO = "[9b6ddad@trunk; built: 2026-01-14T14:27:44Z]";
|
|
20
|
+
var Api = class {
|
|
21
|
+
// @todo(high): types
|
|
22
|
+
headers = {};
|
|
23
|
+
endpoint;
|
|
24
|
+
constructor(configuration2) {
|
|
25
|
+
this.endpoint = configuration2.RUDDERSTACK_API_ENDPOINT;
|
|
26
|
+
this.headers = {
|
|
27
|
+
"User-Agent": `rudderstash/${VERSION}`,
|
|
28
|
+
"Content-Type": "application/json",
|
|
29
|
+
"Authorization": "Basic " + Buffer.from(`${configuration2.RUDDERSTACK_API_USER}:${configuration2.RUDDERSTACK_API_TOKEN}`).toString("base64")
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
async get(path) {
|
|
33
|
+
return (await fetch(this.endpoint + path, { headers: this.headers })).json();
|
|
34
|
+
}
|
|
35
|
+
async post(path, data) {
|
|
36
|
+
return (await fetch(this.endpoint + path, {
|
|
37
|
+
method: "POST",
|
|
38
|
+
headers: this.headers,
|
|
39
|
+
body: JSON.stringify(data)
|
|
40
|
+
})).json();
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
var Transformation = class _Transformation {
|
|
44
|
+
/** @var id A unique transformation ID. */
|
|
45
|
+
id;
|
|
46
|
+
/** @var id A unique transformation version ID. */
|
|
47
|
+
versionId;
|
|
48
|
+
/** @var createdAt The date in UTC when it was created. */
|
|
49
|
+
createdAt;
|
|
50
|
+
/** @var updatedAt The date in UTC when it was updated. */
|
|
51
|
+
updatedAt;
|
|
52
|
+
/** @var name The name for this transformation. */
|
|
53
|
+
name;
|
|
54
|
+
/** @var description The description for this transformation. */
|
|
55
|
+
description;
|
|
56
|
+
/** @var code The transformation source code. */
|
|
57
|
+
code = "";
|
|
58
|
+
/** @var path The local transformation filepath.js. */
|
|
59
|
+
path;
|
|
60
|
+
constructor(metadata = {}) {
|
|
61
|
+
this.id = metadata.id;
|
|
62
|
+
this.versionId = metadata.versionId;
|
|
63
|
+
this.createdAt = metadata.createdAt;
|
|
64
|
+
this.updatedAt = new Date(metadata.updatedAt);
|
|
65
|
+
this.name = metadata.name;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Hydrate a transformation by id from this project.
|
|
69
|
+
*
|
|
70
|
+
* @param id The transformation identifier.
|
|
71
|
+
*
|
|
72
|
+
* @returns A transformation instance or null.
|
|
73
|
+
*/
|
|
74
|
+
static async get(id) {
|
|
75
|
+
const files = await readdir(".");
|
|
76
|
+
for (const file of files.filter((file2) => file2.endsWith(".transformation.js"))) {
|
|
77
|
+
const code = await readFile(file, "utf-8");
|
|
78
|
+
if (!existsSync(`${file}on`)) {
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
const metadata = JSON.parse(await readFile(`${file}on`, "utf-8"));
|
|
82
|
+
if (metadata.id === id) {
|
|
83
|
+
const transformation = new _Transformation(metadata);
|
|
84
|
+
transformation.code = code;
|
|
85
|
+
return transformation;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Provide a hash of this transformation.
|
|
92
|
+
*
|
|
93
|
+
* @return A long hash.
|
|
94
|
+
*/
|
|
95
|
+
hash() {
|
|
96
|
+
return createHash("sha1").update(JSON.stringify({
|
|
97
|
+
id: this.id,
|
|
98
|
+
code: this.code
|
|
99
|
+
// @todo(unknown): do we need the name and description here? author as well maybe?
|
|
100
|
+
})).digest("hex");
|
|
101
|
+
}
|
|
102
|
+
};
|
|
103
|
+
var Project = class _Project {
|
|
104
|
+
/** @var transformations All the transformations in the current project. */
|
|
105
|
+
transformations = [];
|
|
106
|
+
/** @var api The API used. */
|
|
107
|
+
api;
|
|
108
|
+
constructor(configuration2) {
|
|
109
|
+
this.api = new Api(configuration2);
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Read in the current project and return its instance.
|
|
113
|
+
*
|
|
114
|
+
* @returns The project instance.
|
|
115
|
+
*/
|
|
116
|
+
static async read(configuration2) {
|
|
117
|
+
const project = new _Project(configuration2);
|
|
118
|
+
const files = await readdir(".");
|
|
119
|
+
for (const file of files.filter((file2) => file2.endsWith(".transformation.js"))) {
|
|
120
|
+
if (!existsSync(`${file}on`)) {
|
|
121
|
+
const transformation2 = new Transformation();
|
|
122
|
+
transformation2.name = file.split(".transformation.js").at(0);
|
|
123
|
+
transformation2.code = await readFile(file, "utf-8");
|
|
124
|
+
transformation2.path = file;
|
|
125
|
+
project.transformations.push(transformation2);
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
const metadata = JSON.parse(await readFile(`${file}on`, "utf-8"));
|
|
129
|
+
const transformation = await Transformation.get(metadata.id);
|
|
130
|
+
if (transformation) {
|
|
131
|
+
transformation.path = file;
|
|
132
|
+
project.transformations.push(transformation);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
await project.fetch();
|
|
136
|
+
return project;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Fetch changes from upstream into cache.
|
|
140
|
+
*
|
|
141
|
+
* These can be read by diff, etc.
|
|
142
|
+
*/
|
|
143
|
+
async fetch() {
|
|
144
|
+
if (!existsSync(".ruddercache")) {
|
|
145
|
+
await writeFile(".ruddercache", "{}");
|
|
146
|
+
}
|
|
147
|
+
const cache = JSON.parse(await readFile(".ruddercache", "utf8"));
|
|
148
|
+
cache.transformations = (await this.api.get("/transformations")).transformations;
|
|
149
|
+
cache.transformations_fetched_at = Date.now();
|
|
150
|
+
await writeFile(".ruddercache", JSON.stringify(cache));
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* List all cached upstream and local changes.
|
|
154
|
+
*/
|
|
155
|
+
async list() {
|
|
156
|
+
if (!existsSync(".ruddercache")) {
|
|
157
|
+
await writeFile(".ruddercache", "{}");
|
|
158
|
+
}
|
|
159
|
+
let cache = JSON.parse(await readFile(".ruddercache", "utf8"));
|
|
160
|
+
if (cache.transformations === void 0) {
|
|
161
|
+
this.fetch();
|
|
162
|
+
cache = JSON.parse(await readFile(".ruddercache", "utf8"));
|
|
163
|
+
}
|
|
164
|
+
const transformations = cache.transformations;
|
|
165
|
+
const upstream = {};
|
|
166
|
+
const local = {};
|
|
167
|
+
for (const transformation of transformations) {
|
|
168
|
+
const id = transformation.id;
|
|
169
|
+
upstream[id] = new Transformation(transformation);
|
|
170
|
+
upstream[id].code = transformation.code;
|
|
171
|
+
}
|
|
172
|
+
for (const transformation of this.transformations) {
|
|
173
|
+
const id = transformation.id ?? crypto.randomUUID();
|
|
174
|
+
transformation.id = id;
|
|
175
|
+
local[id] || (local[id] = []);
|
|
176
|
+
local[id].push(transformation);
|
|
177
|
+
}
|
|
178
|
+
return {
|
|
179
|
+
transformations_fetched_at: cache.transformations_fetched_at,
|
|
180
|
+
transformations: [upstream, local]
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
// rudderstash.ts
|
|
186
|
+
var configuration = {};
|
|
187
|
+
dotenv.config({
|
|
188
|
+
quiet: true,
|
|
189
|
+
processEnv: configuration,
|
|
190
|
+
override: false
|
|
191
|
+
});
|
|
192
|
+
Object.entries(configuration).forEach(([k, _v]) => {
|
|
193
|
+
if (k in process.env) {
|
|
194
|
+
configuration[k] = process.env[k];
|
|
195
|
+
}
|
|
196
|
+
});
|
|
197
|
+
var cli = await yargs(hideBin(process.argv)).parse();
|
|
198
|
+
var [command, ...subcommand] = cli._;
|
|
199
|
+
switch (command) {
|
|
200
|
+
case "version": {
|
|
201
|
+
console.info(`rudderstash ${VERSION} ${BUILD_INFO}`);
|
|
202
|
+
break;
|
|
203
|
+
}
|
|
204
|
+
case "status": {
|
|
205
|
+
const project = await Project.read(configuration);
|
|
206
|
+
const list = await project.list();
|
|
207
|
+
const [upstream, local] = list.transformations;
|
|
208
|
+
console.info(`Last fetched ${new Date(list.transformations_fetched_at)}
|
|
209
|
+
`);
|
|
210
|
+
console.info("Upstream:");
|
|
211
|
+
if (!Object.keys(upstream).length) {
|
|
212
|
+
console.info(" (none)\n");
|
|
213
|
+
} else for (const transformation of Object.values(upstream)) {
|
|
214
|
+
let status = " ";
|
|
215
|
+
const extra = [];
|
|
216
|
+
if (local[transformation.id] === void 0) {
|
|
217
|
+
status = "+";
|
|
218
|
+
} else {
|
|
219
|
+
const hashChanged = (local[transformation.id] ?? []).reduce(
|
|
220
|
+
(a, c) => a || c.hash() !== transformation.hash(),
|
|
221
|
+
false
|
|
222
|
+
);
|
|
223
|
+
const nameOrDescriptionChanged = (local[transformation.id] ?? []).reduce(
|
|
224
|
+
(a, c) => a || c.name !== transformation.name || c.description !== transformation.description,
|
|
225
|
+
false
|
|
226
|
+
);
|
|
227
|
+
if (hashChanged) {
|
|
228
|
+
status = "~";
|
|
229
|
+
}
|
|
230
|
+
if (nameOrDescriptionChanged) {
|
|
231
|
+
status = "~";
|
|
232
|
+
extra.push("metadata changed");
|
|
233
|
+
}
|
|
234
|
+
if (!hashChanged && !nameOrDescriptionChanged) {
|
|
235
|
+
extra.push("no changes");
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
console.info(
|
|
239
|
+
` ${status} ${transformation.hash().substring(0, 7)} ${transformation.name}` + (extra.length ? ` (${extra.join(", ")})` : "")
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
console.info("\nLocal:");
|
|
243
|
+
if (!Object.keys(local).length) {
|
|
244
|
+
console.info(" (none)\n");
|
|
245
|
+
} else {
|
|
246
|
+
for (const transformations of Object.values(local)) {
|
|
247
|
+
for (const transformation of transformations) {
|
|
248
|
+
let status = " ";
|
|
249
|
+
let extra = [];
|
|
250
|
+
const isDuplicate = transformations.filter((t) => t.id === transformation.id).length > 1;
|
|
251
|
+
if (isDuplicate) {
|
|
252
|
+
status = "!";
|
|
253
|
+
extra.push("duplicate");
|
|
254
|
+
}
|
|
255
|
+
if (upstream[transformation.id] === void 0) {
|
|
256
|
+
status = "+";
|
|
257
|
+
}
|
|
258
|
+
console.info(
|
|
259
|
+
` ${status} ${transformation.hash().substring(0, 7)} ${transformation.name} (${transformation.path})` + (extra.length ? ` (${extra.join(", ")})` : "")
|
|
260
|
+
);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
break;
|
|
265
|
+
}
|
|
266
|
+
case "pull": {
|
|
267
|
+
const project = await Project.read(configuration);
|
|
268
|
+
const list = await project.list();
|
|
269
|
+
const [upstream, local] = list.transformations;
|
|
270
|
+
if (Object.keys(upstream).length < 1) {
|
|
271
|
+
console.info("No transformations upstream, nothing to pull.");
|
|
272
|
+
break;
|
|
273
|
+
}
|
|
274
|
+
for (const transformation of Object.values(upstream)) {
|
|
275
|
+
if (local[transformation.id]) {
|
|
276
|
+
for (const _transformation of Object.values(local[transformation.id])) {
|
|
277
|
+
const filename = _transformation.path;
|
|
278
|
+
console.info(`< Pulling new transformation ${transformation.name} to ${filename}...`);
|
|
279
|
+
await writeFile2(filename, transformation.code);
|
|
280
|
+
console.info(`< Saving metadata to ${filename}on.`);
|
|
281
|
+
delete transformation.code;
|
|
282
|
+
await writeFile2(filename + "on", JSON.stringify(transformation, null, 2));
|
|
283
|
+
}
|
|
284
|
+
} else {
|
|
285
|
+
const filename = transformation.name.split(" ").map((w) => w[0].toUpperCase() + w.substring(1)).join("") + ".transformation.js";
|
|
286
|
+
console.info(`< Pulling new transformation ${transformation.name} to ${filename}...`);
|
|
287
|
+
await writeFile2(filename, transformation.code);
|
|
288
|
+
console.info(`< Saving metadata to ${filename}on.`);
|
|
289
|
+
delete transformation.code;
|
|
290
|
+
await writeFile2(filename + "on", JSON.stringify(transformation, null, 2));
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
break;
|
|
294
|
+
}
|
|
295
|
+
case "push": {
|
|
296
|
+
const project = await Project.read(configuration);
|
|
297
|
+
const list = await project.list();
|
|
298
|
+
const [_upstream, local] = list.transformations;
|
|
299
|
+
if (Object.values(local).length < 1) {
|
|
300
|
+
console.info("No local transformations, nothing to push.");
|
|
301
|
+
break;
|
|
302
|
+
}
|
|
303
|
+
const api = new Api(configuration);
|
|
304
|
+
for (const transformation of Object.values(local)) {
|
|
305
|
+
if (Object.values(transformation).length > 1) {
|
|
306
|
+
console.warn(
|
|
307
|
+
`Conflict! The following local transformations have the same ID: ` + Object.values(transformation).map((t) => t.path).join("\n") + `. Resolve by leaving only one.`.red
|
|
308
|
+
);
|
|
309
|
+
continue;
|
|
310
|
+
}
|
|
311
|
+
const code = await readFile2(transformation[0].path, "utf-8");
|
|
312
|
+
let metadata = {};
|
|
313
|
+
if (transformation[0].versionId === void 0) {
|
|
314
|
+
metadata.id = "";
|
|
315
|
+
metadata.name = transformation[0].path.split(".transformation.js").at(0);
|
|
316
|
+
metadata.description = transformation[0].path;
|
|
317
|
+
} else {
|
|
318
|
+
metadata = JSON.parse(await readFile2(`${transformation[0].path}on`, "utf-8"));
|
|
319
|
+
}
|
|
320
|
+
if (_upstream[transformation[0].id] && transformation[0].hash() === _upstream[transformation[0].id].hash()) {
|
|
321
|
+
console.info(`> Skipping ${metadata.name} transformation. No changes.`);
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
324
|
+
console.info(`> Pushing ${code.length} bytes of code to ${metadata.name} transformation...`);
|
|
325
|
+
const data = await api.post(`/transformations/${metadata.id}?publish=true`, {
|
|
326
|
+
code,
|
|
327
|
+
name: metadata.name,
|
|
328
|
+
language: "javascript",
|
|
329
|
+
// @todo(low): should we support Python? probably not
|
|
330
|
+
description: metadata.description
|
|
331
|
+
});
|
|
332
|
+
if (data.error !== void 0) {
|
|
333
|
+
console.error(data.error);
|
|
334
|
+
continue;
|
|
335
|
+
}
|
|
336
|
+
const refetch = await api.get(`/transformations/${metadata.id}`);
|
|
337
|
+
delete refetch.code;
|
|
338
|
+
await writeFile2(`${transformation[0].path}on`, JSON.stringify(
|
|
339
|
+
{ ...metadata, ...refetch },
|
|
340
|
+
null,
|
|
341
|
+
2
|
|
342
|
+
));
|
|
343
|
+
console.info(`> Published with version ${data.versionId}.`);
|
|
344
|
+
console.info(`< Metadata updated.`);
|
|
345
|
+
}
|
|
346
|
+
break;
|
|
347
|
+
}
|
|
348
|
+
case "diff": {
|
|
349
|
+
const project = await Project.read(configuration);
|
|
350
|
+
const tree = await project.list();
|
|
351
|
+
}
|
|
352
|
+
case "revert": {
|
|
353
|
+
break;
|
|
354
|
+
}
|
|
355
|
+
case "test": {
|
|
356
|
+
const project = await Project.read(configuration);
|
|
357
|
+
const list = await project.list();
|
|
358
|
+
const [_upstream, local] = list.transformations;
|
|
359
|
+
const tests = (await readdir2(".")).filter((f) => f.endsWith(".tests.json"));
|
|
360
|
+
if (tests.length < 1) {
|
|
361
|
+
console.error("No tests to run [*.tests.json].".red);
|
|
362
|
+
process.exit(-1);
|
|
363
|
+
}
|
|
364
|
+
for (const test of tests) {
|
|
365
|
+
const transformation = Object.values(local).flat().find((t) => t.path === test.replace(".tests.json", ".transformation.js"));
|
|
366
|
+
if (transformation === void 0) {
|
|
367
|
+
console.error(`${test} has no matching ${test.replace(".tests.json", ".transformation.js")} file.`.red);
|
|
368
|
+
process.exit(-1);
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
372
|
+
const loaderPath = join(__dirname, "test-loader.js");
|
|
373
|
+
register(loaderPath, import.meta.url);
|
|
374
|
+
for (const test of tests) {
|
|
375
|
+
const transformation = Object.values(local).flat().find((t) => t.path === test.replace(".tests.json", ".transformation.js"));
|
|
376
|
+
console.info(`${transformation.name}:`);
|
|
377
|
+
const cases = JSON.parse(await readFile2(test, "utf8"));
|
|
378
|
+
const importPath = `file://${process.cwd()}/${transformation.path}`;
|
|
379
|
+
const module = await import(importPath);
|
|
380
|
+
for (const _case of cases) {
|
|
381
|
+
const actual = module.transformEvent(_case.input, () => Object.assign({
|
|
382
|
+
destinationId: void 0,
|
|
383
|
+
destinationName: void 0,
|
|
384
|
+
destinationType: void 0,
|
|
385
|
+
sourceId: void 0,
|
|
386
|
+
sourceName: void 0,
|
|
387
|
+
sourceType: void 0
|
|
388
|
+
}, _case.metadata ?? {}));
|
|
389
|
+
const diff = diffJson(
|
|
390
|
+
_case.expected ?? { _is_null: true },
|
|
391
|
+
actual ?? { _is_null: true }
|
|
392
|
+
);
|
|
393
|
+
let status = "FAIL";
|
|
394
|
+
if (diff.length === 1 && !diff[0].added && !diff[0].removed) {
|
|
395
|
+
status = "PASS";
|
|
396
|
+
}
|
|
397
|
+
if (status === "FAIL") {
|
|
398
|
+
console.warn(` ${status} - ${test.split(".json").at(0)}: ${_case.name} `.red);
|
|
399
|
+
for (const changeset of diff) {
|
|
400
|
+
if (!changeset.added && !changeset.removed) {
|
|
401
|
+
console.warn();
|
|
402
|
+
continue;
|
|
403
|
+
}
|
|
404
|
+
if (changeset.added) {
|
|
405
|
+
console.warn(changeset.value.trim().split("\n").map((l) => ` + ${l}`).join("\n"));
|
|
406
|
+
}
|
|
407
|
+
if (changeset.removed) {
|
|
408
|
+
console.warn(changeset.value.trim().split("\n").map((l) => ` - ${l}`).join("\n"));
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
} else {
|
|
412
|
+
console.info(` ${status} - ${test.split(".json").at(0)}: ${_case.name} `.green);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
break;
|
|
417
|
+
}
|
|
418
|
+
default: {
|
|
419
|
+
console.error("Unknown command. Did you mean any of these: status, pull, push, test?".red);
|
|
420
|
+
}
|
|
421
|
+
}
|