rudderstash 0.1.0 → 0.1.1

package/README.md

@@ -39,15 +39,40 @@ Modify.
 rudderstash push
 ```
 
+## Testing Transformations
+
+Testing is simple. Add a `MyTransformation.tests.json` file corresponding to the transformation you'd like to test containing any number of tests.
+
+```json
+[
+  {
+    "name": "My first test",
+    "input": {
+     "ip": "127.0.0.1",
+     "browser": "Chrome",
+    },
+    "expected": {
+      "ip": "127.0.X.X",
+      "browser": "Chrome",
+    }
+  }
+]
+```
+
+Now run all tests against matching transformers like so `rudderstash test`.
+
+*Tip: You can use tests to debug your transformers with anything from good old `console.log` to attaching full fledged debuggers via the `debugger` statement!*
+
 ## Roadmap
 
-- [x] Basic synchronization
-- [ ] Handle renames
 - [ ] TypeScript support
-- [ ] Polish a lot
-- [ ] Testing
 - [ ] Support revisions
 - [ ] Git integration
+- [ ] Add colors
+- [ ] Polish a lot
+- [ ] Test metadata support
+- [ ] Stage or allow partial pushes
+- [ ] Support non-deterministic tests
 
 ## Support & License
 

package/dist/rudderstash.js

@@ -1,7 +1,11 @@
 #!/usr/bin/env node
-"use strict";import{writeFile as h,readFile as g}from"fs/promises";import S from"dotenv";import N from"yargs";import{hideBin as _}from"yargs/helpers";import{existsSync as p}from"fs";import{writeFile as w,readdir as j,readFile as l}from"fs/promises";import{createHash as D}from"crypto";var b="0.1.0",A="[eeb82c8@trunk; built: 2025-12-20T12:01:28Z]",$=class{headers={};endpoint;constructor(r){this.endpoint=r.RUDDERSTACK_API_ENDPOINT,this.headers={"User-Agent":`rudderstash/${b}`,"Content-Type":"application/json",Authorization:"Basic "+Buffer.from(`${r.RUDDERSTACK_API_USER}:${r.RUDDERSTACK_API_TOKEN}`).toString("base64")}}async get(r){return(await fetch(this.endpoint+r,{headers:this.headers})).json()}async post(r,e){return(await fetch(this.endpoint+r,{method:"POST",headers:this.headers,body:JSON.stringify(e)})).json()}},v=class O{id;versionId;createdAt;updatedAt;name;description;code="";path;constructor(e={}){this.id=e.id,this.versionId=e.versionId,this.createdAt=e.createdAt,this.updatedAt=new Date(e.updatedAt),this.name=e.name}static async get(e){const i=await j(".");for(const n of i.filter(t=>t.endsWith(".transformation.js"))){const t=await l(n,"utf-8");if(!p(`${n}on`))continue;const a=JSON.parse(await l(`${n}on`,"utf-8"));if(a.id===e){const o=new O(a);return o.code=t,o}}return null}hash(){return D("sha1").update(JSON.stringify({id:this.id,code:this.code})).digest("hex")}},u=class y{transformations=[];api;constructor(e){this.api=new $(e)}static async read(e){const i=new y(e),n=await j(".");for(const t of n.filter(a=>a.endsWith(".transformation.js"))){if(!p(`${t}on`)){const s=new v;s.name=t.split(".transformation.js").at(0),s.code=await l(t,"utf-8"),s.path=t,i.transformations.push(s);continue}const a=JSON.parse(await l(`${t}on`,"utf-8")),o=await v.get(a.id);o&&(o.path=t,i.transformations.push(o))}return await i.fetch(),i}async fetch(){p(".ruddercache")||await w(".ruddercache","{}");const e=JSON.parse(await l(".ruddercache","utf8"));e.transformations=(await this.api.get("/transformations")).transformations,e.transformations_fetched_at=Date.now(),await w(".ruddercache",JSON.stringify(e))}async list(){p(".ruddercache")||await w(".ruddercache","{}");let e=JSON.parse(await l(".ruddercache","utf8"));e.transformations===void 0&&(this.fetch(),e=JSON.parse(await l(".ruddercache","utf8")));const i=e.transformations,n={},t={};for(const a of i){const o=a.id;n[o]=new v(a),n[o].code=a.code}for(const a of this.transformations){const o=a.id??crypto.randomUUID();a.id=o,t[o]||(t[o]=[]),t[o].push(a)}return{transformations_fetched_at:e.transformations_fetched_at,transformations:[n,t]}}},f={};S.config({quiet:!0,processEnv:f,override:!1}),Object.entries(f).forEach(([r,e])=>{r in process.env&&(f[r]=process.env[r])});var I=await N(_(process.argv)).parse(),[J,...F]=I._;switch(J){case"version":{console.info(`rudderstash ${b} ${A}`);break}case"status":{const e=await(await u.read(f)).list(),[i,n]=e.transformations;if(console.info(`Last fetched ${new Date(e.transformations_fetched_at)}
-`),console.info("Upstream:"),!Object.keys(i).length)console.info(`	(none)
-`);else for(const t of Object.values(i)){let a=" ";const o=[];if(n[t.id]===void 0)a="+";else{const s=(n[t.id]??[]).reduce((d,m)=>d||m.hash()!==t.hash(),!1),c=(n[t.id]??[]).reduce((d,m)=>d||m.name!==t.name||m.description!==t.description,!1);s&&(a="~"),c&&(a="~",o.push("metadata changed")),!s&&!c&&o.push("no changes")}console.info(`	${a} ${t.hash().substring(0,7)} ${t.name}`+(o.length?`	(${o.join(", ")})`:""))}if(console.info(`
-Local:`),!Object.keys(n).length)console.info(`	(none)
-`);else for(const t of Object.values(n))for(const a of t){let o=" ",s=[];t.filter(d=>d.id===a.id).length>1&&(o="!",s.push("duplicate")),i[a.id]===void 0&&(o="+"),console.info(`	${o} ${a.hash().substring(0,7)} ${a.name}	(${a.path})`+(s.length?`	(${s.join(", ")})`:""))}break}case"pull":{const e=await(await u.read(f)).list(),[i,n]=e.transformations;if(Object.keys(i).length<1){console.info("No transformations upstream, nothing to pull.");break}for(const t of Object.values(i))if(n[t.id])for(const a of Object.values(n[t.id])){const o=a.path;console.info(`< Pulling new transformation ${t.name} to ${o}...`),await h(o,t.code),console.info(`< Saving metadata to ${o}on.`),delete t.code,await h(o+"on",JSON.stringify(t,null,2))}else{const a=t.name.split(" ").map(o=>o[0].toUpperCase()+o.substring(1)).join("")+".transformation.js";console.info(`< Pulling new transformation ${t.name} to ${a}...`),await h(a,t.code),console.info(`< Saving metadata to ${a}on.`),delete t.code,await h(a+"on",JSON.stringify(t,null,2))}break}case"push":{const e=await(await u.read(f)).list(),[i,n]=e.transformations;if(Object.values(n).length<1){console.info("No local transformations, nothing to push.");break}const t=new $(f);for(const a of Object.values(n)){if(Object.values(a).length>1){console.warn("Conflict! The following local transformations have the same ID: "+Object.values(a).map(d=>d.path).join(`
-`)+". Resolve by leaving only one.");continue}const o=await g(a[0].path,"utf-8");let s={};a[0].versionId===void 0?(s.id="",s.name=a[0].path.split(".transformation.js").at(0),s.description=a[0].path):s=JSON.parse(await g(`${a[0].path}on`,"utf-8")),console.info(`> Pushing ${o.length} bytes of code to ${s.name} transformation...`);const c=await t.post(`/transformations/${s.id}?publish=true`,{code:o,name:s.name,language:"javascript",description:s.description});if(c.error!==void 0){console.error(c.error);continue}delete c.code,await h(`${a[0].path}on`,JSON.stringify({...s,...c},null,2)),console.info(`> Published with version ${c.versionId}.`),console.info("< Metadata updated.")}break}case"diff":{const e=await(await u.read(f)).list()}case"revert":break;default:console.error("Invalid command")}
+"use strict";import{writeFile as h,readdir as D,readFile as g}from"fs/promises";import"colors";import I from"dotenv";import P from"yargs";import{hideBin as k}from"yargs/helpers";import{diffJson as J}from"diff";import{existsSync as j}from"fs";import{writeFile as $,readdir as O,readFile as l}from"fs/promises";import{createHash as R}from"crypto";var S="0.1.1",E="[e0e30bc@trunk; built: 2025-12-20T15:32:51Z]",y=class{headers={};endpoint;constructor(c){this.endpoint=c.RUDDERSTACK_API_ENDPOINT,this.headers={"User-Agent":`rudderstash/${S}`,"Content-Type":"application/json",Authorization:"Basic "+Buffer.from(`${c.RUDDERSTACK_API_USER}:${c.RUDDERSTACK_API_TOKEN}`).toString("base64")}}async get(c){return(await fetch(this.endpoint+c,{headers:this.headers})).json()}async post(c,a){return(await fetch(this.endpoint+c,{method:"POST",headers:this.headers,body:JSON.stringify(a)})).json()}},b=class N{id;versionId;createdAt;updatedAt;name;description;code="";path;constructor(a={}){this.id=a.id,this.versionId=a.versionId,this.createdAt=a.createdAt,this.updatedAt=new Date(a.updatedAt),this.name=a.name}static async get(a){const r=await O(".");for(const s of r.filter(e=>e.endsWith(".transformation.js"))){const e=await l(s,"utf-8");if(!j(`${s}on`))continue;const t=JSON.parse(await l(`${s}on`,"utf-8"));if(t.id===a){const o=new N(t);return o.code=e,o}}return null}hash(){return R("sha1").update(JSON.stringify({id:this.id,code:this.code})).digest("hex")}},u=class _{transformations=[];api;constructor(a){this.api=new y(a)}static async read(a){const r=new _(a),s=await O(".");for(const e of s.filter(t=>t.endsWith(".transformation.js"))){if(!j(`${e}on`)){const n=new b;n.name=e.split(".transformation.js").at(0),n.code=await l(e,"utf-8"),n.path=e,r.transformations.push(n);continue}const t=JSON.parse(await l(`${e}on`,"utf-8")),o=await b.get(t.id);o&&(o.path=e,r.transformations.push(o))}return await r.fetch(),r}async fetch(){j(".ruddercache")||await $(".ruddercache","{}");const a=JSON.parse(await l(".ruddercache","utf8"));a.transformations=(await this.api.get("/transformations")).transformations,a.transformations_fetched_at=Date.now(),await $(".ruddercache",JSON.stringify(a))}async list(){j(".ruddercache")||await $(".ruddercache","{}");let a=JSON.parse(await l(".ruddercache","utf8"));a.transformations===void 0&&(this.fetch(),a=JSON.parse(await l(".ruddercache","utf8")));const r=a.transformations,s={},e={};for(const t of r){const o=t.id;s[o]=new b(t),s[o].code=t.code}for(const t of this.transformations){const o=t.id??crypto.randomUUID();t.id=o,e[o]||(e[o]=[]),e[o].push(t)}return{transformations_fetched_at:a.transformations_fetched_at,transformations:[s,e]}}},d={};I.config({quiet:!0,processEnv:d,override:!1}),Object.entries(d).forEach(([c,a])=>{c in process.env&&(d[c]=process.env[c])});var T=await P(k(process.argv)).parse(),[U,...H]=T._;switch(U){case"version":{console.info(`rudderstash ${S} ${E}`);break}case"status":{const a=await(await u.read(d)).list(),[r,s]=a.transformations;if(console.info(`Last fetched ${new Date(a.transformations_fetched_at)}
+`),console.info("Upstream:"),!Object.keys(r).length)console.info(`	(none)
+`);else for(const e of Object.values(r)){let t=" ";const o=[];if(s[e.id]===void 0)t="+";else{const n=(s[e.id]??[]).reduce((f,p)=>f||p.hash()!==e.hash(),!1),i=(s[e.id]??[]).reduce((f,p)=>f||p.name!==e.name||p.description!==e.description,!1);n&&(t="~"),i&&(t="~",o.push("metadata changed")),!n&&!i&&o.push("no changes")}console.info(`	${t} ${e.hash().substring(0,7)} ${e.name}`+(o.length?`	(${o.join(", ")})`:""))}if(console.info(`
+Local:`),!Object.keys(s).length)console.info(`	(none)
+`);else for(const e of Object.values(s))for(const t of e){let o=" ",n=[];e.filter(f=>f.id===t.id).length>1&&(o="!",n.push("duplicate")),r[t.id]===void 0&&(o="+"),console.info(`	${o} ${t.hash().substring(0,7)} ${t.name}	(${t.path})`+(n.length?`	(${n.join(", ")})`:""))}break}case"pull":{const a=await(await u.read(d)).list(),[r,s]=a.transformations;if(Object.keys(r).length<1){console.info("No transformations upstream, nothing to pull.");break}for(const e of Object.values(r))if(s[e.id])for(const t of Object.values(s[e.id])){const o=t.path;console.info(`< Pulling new transformation ${e.name} to ${o}...`),await h(o,e.code),console.info(`< Saving metadata to ${o}on.`),delete e.code,await h(o+"on",JSON.stringify(e,null,2))}else{const t=e.name.split(" ").map(o=>o[0].toUpperCase()+o.substring(1)).join("")+".transformation.js";console.info(`< Pulling new transformation ${e.name} to ${t}...`),await h(t,e.code),console.info(`< Saving metadata to ${t}on.`),delete e.code,await h(t+"on",JSON.stringify(e,null,2))}break}case"push":{const a=await(await u.read(d)).list(),[r,s]=a.transformations;if(Object.values(s).length<1){console.info("No local transformations, nothing to push.");break}const e=new y(d);for(const t of Object.values(s)){if(Object.values(t).length>1){console.warn("Conflict! The following local transformations have the same ID: "+Object.values(t).map(f=>f.path).join(`
+`)+". Resolve by leaving only one.".red);continue}const o=await g(t[0].path,"utf-8");let n={};t[0].versionId===void 0?(n.id="",n.name=t[0].path.split(".transformation.js").at(0),n.description=t[0].path):n=JSON.parse(await g(`${t[0].path}on`,"utf-8")),console.info(`> Pushing ${o.length} bytes of code to ${n.name} transformation...`);const i=await e.post(`/transformations/${n.id}?publish=true`,{code:o,name:n.name,language:"javascript",description:n.description});if(i.error!==void 0){console.error(i.error);continue}delete i.code,await h(`${t[0].path}on`,JSON.stringify({...n,...i},null,2)),console.info(`> Published with version ${i.versionId}.`),console.info("< Metadata updated.")}break}case"diff":{const a=await(await u.read(d)).list()}case"revert":break;case"test":{const a=await(await u.read(d)).list(),[r,s]=a.transformations,e=(await D(".")).filter(t=>t.endsWith(".tests.json"));e.length<1&&(console.error("No tests to run [*.tests.json].".red),process.exit(-1));for(const t of e)Object.values(s).flat().find(n=>n.path===t.replace(".tests.json",".transformation.js"))===void 0&&(console.error(`${t} has no matching ${t.replace(".tests.json",".transformation.js")} file.`.red),process.exit(-1));process.removeAllListeners("warning");for(const t of e){const o=Object.values(s).flat().find(i=>i.path===t.replace(".tests.json",".transformation.js"));console.info(`${o.name}:`);const n=JSON.parse(await g(t,"utf8"));for(const i of n){let f=" ERR";const A=await import(`file://${process.cwd()}/${o.path}`),w=J(i.expected,A.transformEvent(i.input));switch(f="FAIL",w.length===1&&!w[0].added&&!w[0].removed&&(f="PASS"),f){case"FAIL":console.warn(`	${f} - ${t.split(".json").at(0)}: ${i.name}	`.red);for(const m of w){if(!m.added&&!m.removed){console.warn();continue}m.added&&console.warn(m.value.trim().split(`
+`).map(v=>`		+ ${v}`).join(`
+`)),m.removed&&console.warn(m.value.trim().split(`
+`).map(v=>`		- ${v}`).join(`
+`))}break;case"PASS":console.info(`	${f} - ${t.split(".json").at(0)}: ${i.name}	`.green);break;default:console.info(`	${f} - ${t.split(".json").at(0)}: ${i.name}	`.bgRed);break}}}break}default:console.error("Unknown command. Did you mean any of these: status, pull, push, test?".red)}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rudderstash",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Rudderstack transformation version control, deployment and testing.",
   "keywords": [
     "rudderstack"
@@ -20,9 +20,10 @@
     "rudderstash": "./dist/rudderstash.js"
   },
   "scripts": {
-    "build": "esbuild rudderstash.ts --bundle --platform=node --format=esm --outfile=dist/rudderstash.js --banner:js='#!/usr/bin/env node' --packages=external",
-    "build:minify": "esbuild rudderstash.ts --bundle --platform=node --format=esm --outfile=dist/rudderstash.js --banner:js='#!/usr/bin/env node' --packages=external --minify",
+    "build": "esbuild rudderstash.ts --bundle --platform=node --format=esm --outfile=dist/rudderstash.js --banner:js='#!/usr/bin/env node' --packages=external --external:*.transformation.js",
+    "build:minify": "esbuild rudderstash.ts --bundle --platform=node --format=esm --outfile=dist/rudderstash.js --banner:js='#!/usr/bin/env node' --packages=external --minify --external:*.transformation.js",
     "dev": "tsx rudderstash.ts",
+    "check": "tsc",
     "debug": "tsx --inspect-brk rudderstash.ts",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
@@ -37,6 +38,7 @@
     "typescript": "^5.9.3"
   },
   "dependencies": {
+    "colors": "^1.4.0",
     "diff": "^8.0.2",
     "dotenv": "^17.2.3",
     "yargs": "^18.0.0"