rudderstash 0.1.0 → 0.1.2

package/README.md

@@ -19,7 +19,7 @@ mkdir project && cd project
 
 A read-write Rudderstack API token is required for synchronization. This can either be [Personal Access Token](https://app.rudderstack.com/profile) or a [Service Access Token](https://app.rudderstack.com/organization?tab=service_access_tokens). Set the `RUDDERSTACK_API_TOKEN` in the .env file in the project root or supply it via environment variables. See .env.example, and remember to **never commit your .env file**.
 
-The `RUDDERSTACK_API_USER` (matches your account email) and `RUDDERSTACK_API_ENDPOINT` should also be supplied.
+The `RUDDERSTACK_API_USER` (matches your account email) and `RUDDERSTACK_API_ENDPOINT` should also be supplied (either `https://api.rudderstack.com` for US-based data planes or `https://api.eu.rudderstack.com` for EU ones).
 
 ```bash
 rudderstash status
@@ -39,15 +39,43 @@ Modify.
 rudderstash push
 ```
 
+## Testing Transformations
+
+Testing is simple. Add a `MyTransformation.tests.json` file corresponding to the transformation you'd like to test containing any number of tests.
+
+```json
+[
+  {
+    "name": "My first test",
+    "input": {
+     "ip": "127.0.0.1",
+     "browser": "Chrome",
+    },
+    "expected": {
+      "ip": "127.0.X.X",
+      "browser": "Chrome",
+    },
+    "metadata": {
+      "destinationId": "....",
+    },
+  }
+]
+```
+
+Now run all tests against matching transformers like so `rudderstash test`.
+
+*Tip: You can use tests to debug your transformers with anything from good old `console.log` to attaching full fledged debuggers via the `debugger` statement!*
+
 ## Roadmap
 
-- [x] Basic synchronization
-- [ ] Handle renames
 - [ ] TypeScript support
-- [ ] Polish a lot
-- [ ] Testing
 - [ ] Support revisions
 - [ ] Git integration
+- [ ] Add colors
+- [ ] Polish a lot
+- [ ] Test metadata support
+- [ ] Stage or allow partial pushes
+- [ ] Support non-deterministic tests
 
 ## Support & License
 

package/dist/rudderstash.js

@@ -1,7 +1,11 @@
 #!/usr/bin/env node
-"use strict";import{writeFile as h,readFile as g}from"fs/promises";import S from"dotenv";import N from"yargs";import{hideBin as _}from"yargs/helpers";import{existsSync as p}from"fs";import{writeFile as w,readdir as j,readFile as l}from"fs/promises";import{createHash as D}from"crypto";var b="0.1.0",A="[eeb82c8@trunk; built: 2025-12-20T12:01:28Z]",$=class{headers={};endpoint;constructor(r){this.endpoint=r.RUDDERSTACK_API_ENDPOINT,this.headers={"User-Agent":`rudderstash/${b}`,"Content-Type":"application/json",Authorization:"Basic "+Buffer.from(`${r.RUDDERSTACK_API_USER}:${r.RUDDERSTACK_API_TOKEN}`).toString("base64")}}async get(r){return(await fetch(this.endpoint+r,{headers:this.headers})).json()}async post(r,e){return(await fetch(this.endpoint+r,{method:"POST",headers:this.headers,body:JSON.stringify(e)})).json()}},v=class O{id;versionId;createdAt;updatedAt;name;description;code="";path;constructor(e={}){this.id=e.id,this.versionId=e.versionId,this.createdAt=e.createdAt,this.updatedAt=new Date(e.updatedAt),this.name=e.name}static async get(e){const i=await j(".");for(const n of i.filter(t=>t.endsWith(".transformation.js"))){const t=await l(n,"utf-8");if(!p(`${n}on`))continue;const a=JSON.parse(await l(`${n}on`,"utf-8"));if(a.id===e){const o=new O(a);return o.code=t,o}}return null}hash(){return D("sha1").update(JSON.stringify({id:this.id,code:this.code})).digest("hex")}},u=class y{transformations=[];api;constructor(e){this.api=new $(e)}static async read(e){const i=new y(e),n=await j(".");for(const t of n.filter(a=>a.endsWith(".transformation.js"))){if(!p(`${t}on`)){const s=new v;s.name=t.split(".transformation.js").at(0),s.code=await l(t,"utf-8"),s.path=t,i.transformations.push(s);continue}const a=JSON.parse(await l(`${t}on`,"utf-8")),o=await v.get(a.id);o&&(o.path=t,i.transformations.push(o))}return await i.fetch(),i}async fetch(){p(".ruddercache")||await w(".ruddercache","{}");const e=JSON.parse(await l(".ruddercache","utf8"));e.transformations=(await this.api.get("/transformations")).transformations,e.transformations_fetched_at=Date.now(),await w(".ruddercache",JSON.stringify(e))}async list(){p(".ruddercache")||await w(".ruddercache","{}");let e=JSON.parse(await l(".ruddercache","utf8"));e.transformations===void 0&&(this.fetch(),e=JSON.parse(await l(".ruddercache","utf8")));const i=e.transformations,n={},t={};for(const a of i){const o=a.id;n[o]=new v(a),n[o].code=a.code}for(const a of this.transformations){const o=a.id??crypto.randomUUID();a.id=o,t[o]||(t[o]=[]),t[o].push(a)}return{transformations_fetched_at:e.transformations_fetched_at,transformations:[n,t]}}},f={};S.config({quiet:!0,processEnv:f,override:!1}),Object.entries(f).forEach(([r,e])=>{r in process.env&&(f[r]=process.env[r])});var I=await N(_(process.argv)).parse(),[J,...F]=I._;switch(J){case"version":{console.info(`rudderstash ${b} ${A}`);break}case"status":{const e=await(await u.read(f)).list(),[i,n]=e.transformations;if(console.info(`Last fetched ${new Date(e.transformations_fetched_at)}
+"use strict";import{writeFile as p,readdir as I,readFile as g}from"fs/promises";import"colors";import D from"dotenv";import k from"yargs";import{hideBin as P}from"yargs/helpers";import{diffJson as J}from"diff";import{existsSync as v}from"fs";import{writeFile as $,readdir as O,readFile as l}from"fs/promises";import{createHash as T}from"crypto";var S="0.1.2",R="[24ac473@trunk; built: 2025-12-25T11:10:26Z]",y=class{headers={};endpoint;constructor(c){this.endpoint=c.RUDDERSTACK_API_ENDPOINT,this.headers={"User-Agent":`rudderstash/${S}`,"Content-Type":"application/json",Authorization:"Basic "+Buffer.from(`${c.RUDDERSTACK_API_USER}:${c.RUDDERSTACK_API_TOKEN}`).toString("base64")}}async get(c){return(await fetch(this.endpoint+c,{headers:this.headers})).json()}async post(c,a){return(await fetch(this.endpoint+c,{method:"POST",headers:this.headers,body:JSON.stringify(a)})).json()}},b=class _{id;versionId;createdAt;updatedAt;name;description;code="";path;constructor(a={}){this.id=a.id,this.versionId=a.versionId,this.createdAt=a.createdAt,this.updatedAt=new Date(a.updatedAt),this.name=a.name}static async get(a){const i=await O(".");for(const s of i.filter(o=>o.endsWith(".transformation.js"))){const o=await l(s,"utf-8");if(!v(`${s}on`))continue;const t=JSON.parse(await l(`${s}on`,"utf-8"));if(t.id===a){const e=new _(t);return e.code=o,e}}return null}hash(){return T("sha1").update(JSON.stringify({id:this.id,code:this.code})).digest("hex")}},u=class N{transformations=[];api;constructor(a){this.api=new y(a)}static async read(a){const i=new N(a),s=await O(".");for(const o of s.filter(t=>t.endsWith(".transformation.js"))){if(!v(`${o}on`)){const n=new b;n.name=o.split(".transformation.js").at(0),n.code=await l(o,"utf-8"),n.path=o,i.transformations.push(n);continue}const t=JSON.parse(await l(`${o}on`,"utf-8")),e=await b.get(t.id);e&&(e.path=o,i.transformations.push(e))}return await i.fetch(),i}async fetch(){v(".ruddercache")||await $(".ruddercache","{}");const a=JSON.parse(await l(".ruddercache","utf8"));a.transformations=(await this.api.get("/transformations")).transformations,a.transformations_fetched_at=Date.now(),await $(".ruddercache",JSON.stringify(a))}async list(){v(".ruddercache")||await $(".ruddercache","{}");let a=JSON.parse(await l(".ruddercache","utf8"));a.transformations===void 0&&(this.fetch(),a=JSON.parse(await l(".ruddercache","utf8")));const i=a.transformations,s={},o={};for(const t of i){const e=t.id;s[e]=new b(t),s[e].code=t.code}for(const t of this.transformations){const e=t.id??crypto.randomUUID();t.id=e,o[e]||(o[e]=[]),o[e].push(t)}return{transformations_fetched_at:a.transformations_fetched_at,transformations:[s,o]}}},d={};D.config({quiet:!0,processEnv:d,override:!1}),Object.entries(d).forEach(([c,a])=>{c in process.env&&(d[c]=process.env[c])});var E=await k(P(process.argv)).parse(),[U,...M]=E._;switch(U){case"version":{console.info(`rudderstash ${S} ${R}`);break}case"status":{const a=await(await u.read(d)).list(),[i,s]=a.transformations;if(console.info(`Last fetched ${new Date(a.transformations_fetched_at)}
 `),console.info("Upstream:"),!Object.keys(i).length)console.info(`	(none)
-`);else for(const t of Object.values(i)){let a=" ";const o=[];if(n[t.id]===void 0)a="+";else{const s=(n[t.id]??[]).reduce((d,m)=>d||m.hash()!==t.hash(),!1),c=(n[t.id]??[]).reduce((d,m)=>d||m.name!==t.name||m.description!==t.description,!1);s&&(a="~"),c&&(a="~",o.push("metadata changed")),!s&&!c&&o.push("no changes")}console.info(`	${a} ${t.hash().substring(0,7)} ${t.name}`+(o.length?`	(${o.join(", ")})`:""))}if(console.info(`
-Local:`),!Object.keys(n).length)console.info(`	(none)
-`);else for(const t of Object.values(n))for(const a of t){let o=" ",s=[];t.filter(d=>d.id===a.id).length>1&&(o="!",s.push("duplicate")),i[a.id]===void 0&&(o="+"),console.info(`	${o} ${a.hash().substring(0,7)} ${a.name}	(${a.path})`+(s.length?`	(${s.join(", ")})`:""))}break}case"pull":{const e=await(await u.read(f)).list(),[i,n]=e.transformations;if(Object.keys(i).length<1){console.info("No transformations upstream, nothing to pull.");break}for(const t of Object.values(i))if(n[t.id])for(const a of Object.values(n[t.id])){const o=a.path;console.info(`< Pulling new transformation ${t.name} to ${o}...`),await h(o,t.code),console.info(`< Saving metadata to ${o}on.`),delete t.code,await h(o+"on",JSON.stringify(t,null,2))}else{const a=t.name.split(" ").map(o=>o[0].toUpperCase()+o.substring(1)).join("")+".transformation.js";console.info(`< Pulling new transformation ${t.name} to ${a}...`),await h(a,t.code),console.info(`< Saving metadata to ${a}on.`),delete t.code,await h(a+"on",JSON.stringify(t,null,2))}break}case"push":{const e=await(await u.read(f)).list(),[i,n]=e.transformations;if(Object.values(n).length<1){console.info("No local transformations, nothing to push.");break}const t=new $(f);for(const a of Object.values(n)){if(Object.values(a).length>1){console.warn("Conflict! The following local transformations have the same ID: "+Object.values(a).map(d=>d.path).join(`
-`)+". Resolve by leaving only one.");continue}const o=await g(a[0].path,"utf-8");let s={};a[0].versionId===void 0?(s.id="",s.name=a[0].path.split(".transformation.js").at(0),s.description=a[0].path):s=JSON.parse(await g(`${a[0].path}on`,"utf-8")),console.info(`> Pushing ${o.length} bytes of code to ${s.name} transformation...`);const c=await t.post(`/transformations/${s.id}?publish=true`,{code:o,name:s.name,language:"javascript",description:s.description});if(c.error!==void 0){console.error(c.error);continue}delete c.code,await h(`${a[0].path}on`,JSON.stringify({...s,...c},null,2)),console.info(`> Published with version ${c.versionId}.`),console.info("< Metadata updated.")}break}case"diff":{const e=await(await u.read(f)).list()}case"revert":break;default:console.error("Invalid command")}
+`);else for(const o of Object.values(i)){let t=" ";const e=[];if(s[o.id]===void 0)t="+";else{const n=(s[o.id]??[]).reduce((f,h)=>f||h.hash()!==o.hash(),!1),r=(s[o.id]??[]).reduce((f,h)=>f||h.name!==o.name||h.description!==o.description,!1);n&&(t="~"),r&&(t="~",e.push("metadata changed")),!n&&!r&&e.push("no changes")}console.info(`	${t} ${o.hash().substring(0,7)} ${o.name}`+(e.length?`	(${e.join(", ")})`:""))}if(console.info(`
+Local:`),!Object.keys(s).length)console.info(`	(none)
+`);else for(const o of Object.values(s))for(const t of o){let e=" ",n=[];o.filter(f=>f.id===t.id).length>1&&(e="!",n.push("duplicate")),i[t.id]===void 0&&(e="+"),console.info(`	${e} ${t.hash().substring(0,7)} ${t.name}	(${t.path})`+(n.length?`	(${n.join(", ")})`:""))}break}case"pull":{const a=await(await u.read(d)).list(),[i,s]=a.transformations;if(Object.keys(i).length<1){console.info("No transformations upstream, nothing to pull.");break}for(const o of Object.values(i))if(s[o.id])for(const t of Object.values(s[o.id])){const e=t.path;console.info(`< Pulling new transformation ${o.name} to ${e}...`),await p(e,o.code),console.info(`< Saving metadata to ${e}on.`),delete o.code,await p(e+"on",JSON.stringify(o,null,2))}else{const t=o.name.split(" ").map(e=>e[0].toUpperCase()+e.substring(1)).join("")+".transformation.js";console.info(`< Pulling new transformation ${o.name} to ${t}...`),await p(t,o.code),console.info(`< Saving metadata to ${t}on.`),delete o.code,await p(t+"on",JSON.stringify(o,null,2))}break}case"push":{const a=await(await u.read(d)).list(),[i,s]=a.transformations;if(Object.values(s).length<1){console.info("No local transformations, nothing to push.");break}const o=new y(d);for(const t of Object.values(s)){if(Object.values(t).length>1){console.warn("Conflict! The following local transformations have the same ID: "+Object.values(t).map(f=>f.path).join(`
+`)+". Resolve by leaving only one.".red);continue}const e=await g(t[0].path,"utf-8");let n={};if(t[0].versionId===void 0?(n.id="",n.name=t[0].path.split(".transformation.js").at(0),n.description=t[0].path):n=JSON.parse(await g(`${t[0].path}on`,"utf-8")),i[t[0].id]&&t[0].hash()===i[t[0].id].hash()){console.info(`> Skipping ${n.name} transformation. No changes.`);continue}console.info(`> Pushing ${e.length} bytes of code to ${n.name} transformation...`);const r=await o.post(`/transformations/${n.id}?publish=true`,{code:e,name:n.name,language:"javascript",description:n.description});if(r.error!==void 0){console.error(r.error);continue}delete r.code,await p(`${t[0].path}on`,JSON.stringify({...n,...r},null,2)),console.info(`> Published with version ${r.versionId}.`),console.info("< Metadata updated.")}break}case"diff":{const a=await(await u.read(d)).list()}case"revert":break;case"test":{const a=await(await u.read(d)).list(),[i,s]=a.transformations,o=(await I(".")).filter(t=>t.endsWith(".tests.json"));o.length<1&&(console.error("No tests to run [*.tests.json].".red),process.exit(-1));for(const t of o)Object.values(s).flat().find(n=>n.path===t.replace(".tests.json",".transformation.js"))===void 0&&(console.error(`${t} has no matching ${t.replace(".tests.json",".transformation.js")} file.`.red),process.exit(-1));process.removeAllListeners("warning");for(const t of o){const e=Object.values(s).flat().find(r=>r.path===t.replace(".tests.json",".transformation.js"));console.info(`${e.name}:`);const n=JSON.parse(await g(t,"utf8"));for(const r of n){let f=" ERR";const A=(await import(`file://${process.cwd()}/${e.path}`)).transformEvent(r.input,()=>Object.assign({destinationId:void 0,destinationName:void 0,destinationType:void 0,sourceId:void 0,sourceName:void 0,sourceType:void 0},r.metadata??{})),w=J(r.expected??{_is_null:!0},A??{_is_null:!0});switch(f="FAIL",w.length===1&&!w[0].added&&!w[0].removed&&(f="PASS"),f){case"FAIL":console.warn(`	${f} - ${t.split(".json").at(0)}: ${r.name}	`.red);for(const m of w){if(!m.added&&!m.removed){console.warn();continue}m.added&&console.warn(m.value.trim().split(`
+`).map(j=>`		+ ${j}`).join(`
+`)),m.removed&&console.warn(m.value.trim().split(`
+`).map(j=>`		- ${j}`).join(`
+`))}break;case"PASS":console.info(`	${f} - ${t.split(".json").at(0)}: ${r.name}	`.green);break;default:console.info(`	${f} - ${t.split(".json").at(0)}: ${r.name}	`.bgRed);break}}}break}default:console.error("Unknown command. Did you mean any of these: status, pull, push, test?".red)}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudderstash",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Rudderstack transformation version control, deployment and testing.",
   "keywords": [
     "rudderstack"
@@ -20,9 +20,10 @@
     "rudderstash": "./dist/rudderstash.js"
   },
   "scripts": {
-    "build": "esbuild rudderstash.ts --bundle --platform=node --format=esm --outfile=dist/rudderstash.js --banner:js='#!/usr/bin/env node' --packages=external",
-    "build:minify": "esbuild rudderstash.ts --bundle --platform=node --format=esm --outfile=dist/rudderstash.js --banner:js='#!/usr/bin/env node' --packages=external --minify",
+    "build": "esbuild rudderstash.ts --bundle --platform=node --format=esm --outfile=dist/rudderstash.js --banner:js='#!/usr/bin/env node' --packages=external --external:*.transformation.js",
+    "build:minify": "esbuild rudderstash.ts --bundle --platform=node --format=esm --outfile=dist/rudderstash.js --banner:js='#!/usr/bin/env node' --packages=external --minify --external:*.transformation.js",
     "dev": "tsx rudderstash.ts",
+    "check": "tsc",
     "debug": "tsx --inspect-brk rudderstash.ts",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
@@ -37,6 +38,7 @@
     "typescript": "^5.9.3"
   },
   "dependencies": {
+    "colors": "^1.4.0",
     "diff": "^8.0.2",
     "dotenv": "^17.2.3",
     "yargs": "^18.0.0"