sptc 0.0.0 → 0.0.2

package/engine/index.js

@@ -0,0 +1,64 @@
+const Compiler=require('./compiler')
+const Interpreter=require('./interpreter')
+const CompilerMacro=require('./compiler-macro')
+const InterpreterMacro=require('./interpreter-macro')
+
+const {md5}=require('../utils')
+
+function executeSptcFile(filename, payload, option={}) {
+	const {__DEV__, macroOption, mockFileContent}=option
+
+	// using macro preprocessor or provides a mock content will critically degrade the performance
+	// if you really want to use them, specified the option.__DEV__ as true
+	const _option=__DEV__!==true? {}: {
+	  mockFileContent,
+		contentWrapper: macroOption?
+		  content=>executeSptcMacroFile(filename, macroOption, content):
+		  undefined,
+		cacheKeyWrapper: (!mockFileContent && !macroOption)?
+		  undefined:
+		  filename=>md5(filename+'\n'+[mockFileContent || '', ...(macroOption.defs || [])].join('\n')),
+  }
+
+	const [ctx0, vm]=Compiler.compileSptcFile(filename, _option)
+  const [ctx, priv]=Interpreter.buildContext({...payload, ...ctx0}, option)
+  Interpreter.executeVm(vm, ctx, priv)
+}
+
+function executeSptcFileEx(...x) {
+	const e=new Promise((resolve, reject)=>{
+  	const {end, onError, readAsBinary}=(x[2]=x[2] || {})
+  	const buf=[]
+  	x[2].write=(...x)=>{
+  		buf.push(...x)
+  	}
+  	x[2].end=_=>{
+  		end && end()
+  		resolve(readAsBinary? Buffer.concat(buf): buf.join(''))
+  	}
+  	x[2].onError=e=>{
+  		onError && onError(e)
+  		reject(e)
+  	}
+	  executeSptcFile(...x)
+	})
+	e.catch(_=>null)
+	return e
+}
+
+function executeSptcMacroFile(filename, config, mockContent) {
+	const ast=CompilerMacro.compileSptcMacroFile(filename, mockContent)
+	const ctx=InterpreterMacro.buildMacroContext({filename, ...config})
+	return InterpreterMacro.executeMacroContext(ctx, ast)
+}
+
+module.exports={
+	...Compiler,
+	...Interpreter,
+	...CompilerMacro,
+	...InterpreterMacro,
+	executeSptcFile,
+	executeSptcFileEx,
+	executeSptcMacroFile,
+	version: require('../package.json').version
+}

package/engine/interpreter-macro.js

@@ -0,0 +1,77 @@
+const {compileSptcMacroFile, MACRO_TYPES}=require('./compiler-macro')
+
+const {
+  O_IFDEF,
+  O_STR,
+  O_DEFINE_CONST,
+  O_DEFINE_CALL,
+  O_CALL_CONST,
+  O_CALL_DEFINE,
+  O_DEF,
+  O_INCLUDE,
+  O_UNDEF,
+}=MACRO_TYPES
+
+function buildMacroContext({filename, ...inherits}) {
+  const ctx=Object.assign({
+    defs: new Set,
+    defines: {},
+    filename: filename || '',
+  }, inherits || {})
+  if(filename) {
+    ctx.filename=filename
+  }
+  if(Array.isArray(ctx.defs)) {
+    ctx.defs=new Set(ctx.defs)
+  }
+  return ctx
+}
+
+function execute(ctx, ast) {
+  let ret=''
+  for(let i=0; i<ast.length; i++) {
+    const p=ast[i]
+    if(p.type===O_IFDEF) {
+      const {match, consequent, alternate}=p
+      const sub_tree=ctx.defs.has(match)? consequent: alternate
+      ret+=execute(ctx, sub_tree)
+    }else if(p.type===O_STR) {
+      ret+=p.str
+    }else if(p.type===O_DEFINE_CONST) {
+      ctx.defines[p.cname]=p.cvalue
+    }else if(p.type===O_DEFINE_CALL) {
+      ctx.defines[p.fname]=p.fcall
+    }else if([O_CALL_CONST, O_CALL_DEFINE].includes(p.type)) {
+      if(!(p.call in ctx.defines)) {
+        ret+=p.source
+      }else{
+        if(p.type===O_CALL_CONST) {
+          ret+=ctx.defines[p.call]
+        }else{
+          ret+=ctx.defines[p.call](...p.argv)
+        }
+      }
+    }else if(p.type===O_DEF) {
+      ctx.defs.add(p.def)
+    }else if(p.type===O_UNDEF) {
+      ctx.defs.delete(p.def)
+    }else if(p.type===O_INCLUDE) {
+      const filename=path.resolve(ctx.filename+'/..', p.include)
+      const nctx=Object.assign({}, ctx, {filename})
+      ret+=executeContext(nctx)
+    }else{
+      throw new Error('unsupported node: '+JSON.stringify(p))
+    }
+  }
+  return ret
+}
+
+function executeMacroContext(ctx, ast) {
+  const {filename}=ctx
+  return execute(ctx, ast || compileSptcMacroFile(filename))
+}
+
+module.exports={
+  buildMacroContext,
+  executeMacroContext,
+}

package/engine/interpreter.js

@@ -0,0 +1,291 @@
+const _module=require('module')
+const vm=require('vm')
+const path=require('path')
+const Compiler=require('./compiler')
+const {generate_uuid}=require('../utils')
+
+/*
+  const tt={x: 2, b: [2, 5], c: _=>2, d: /xx/ig}
+  tt.y=tt
+  tt.qq=[{r: tt}]
+  const r1={zzz: 44}
+  r1.rrr=r1
+  tt.H=r1
+  console.log(var_dump(tt))
+ */
+function var_dump(...x) {
+  return x.map(_var_dump).join('\n')
+}
+function _var_dump(x) {
+  let i=new WeakSet, b=0, K=generate_uuid('VAR_DUMP__'), ls=[]
+  JSON.stringify(x, (k, v)=>{
+    if(v && typeof v==='object') {
+      if(!i.has(v)) {
+        v[K]=b++
+        i.add(v)
+        ls.push(v)
+      }else return 0
+    }
+    return v
+  })
+
+  i=new WeakSet
+  b=0
+  ls=[]
+
+  const e=JSON.stringify(x, (k, v)=>{
+    if(v && typeof v==='object') {
+      if(!i.has(v)) {
+        v[K]=b++
+        i.add(v)
+        ls.push(v)
+      }else{
+        return `[Circular] #`+v[K]
+      }
+    }
+    if(k===K) {
+      return '[*Ref] #'+v
+    }
+    if(typeof v==='function') {
+      return `[Function] ${v.toString()}`
+    }
+    if(v && v.constructor===RegExp) {
+      return `[RegExp] ${v.toString()}`
+    }
+    return v
+  }, 2)
+
+  for(let o of ls) {
+    delete o[K]
+  }
+
+  return e && e.replace(new RegExp(K, 'g'), '__ref_key__')
+}
+
+function buildGlobal() {
+  const ctx={
+    // nodejs
+    setTimeout,
+    setInterval,
+    clearTimeout,
+    clearInterval,
+    queueMicrotask,
+    clearImmediate,
+    setImmediate,
+
+    console,
+    Buffer,
+    process,
+
+    TextDecoder,
+    TextEncoder,
+    URL,
+    URLSearchParams,
+
+    Promise,
+    // require
+
+    // v8
+    JSON,
+    RegExp,
+    Object,
+    Array,
+    String,
+    Number,
+    Boolean,
+    Date,
+    Math,
+    Set, WeakSet,
+    Map, WeakMap,
+    Proxy,
+    Symbol,
+    Error,
+    Function,
+    // eval,
+
+    escape, unescape,
+    encodeURI, decodeURI,
+    encodeURIComponent, decodeURIComponent,
+    isNaN, isFinite,
+    parseInt, parseFloat,
+  }
+  ctx.global=ctx
+  ctx.globalThis=ctx
+  return ctx
+}
+
+let _patched=false
+function securityPatch() {
+  if(_patched) return;
+  _patched=true
+  const sfunc=(new vm.Script(`(_=>null).constructor`)).runInNewContext({})
+  parseInt.constructor.prototype.constructor=sfunc
+  global.Function=sfunc
+  global.global=buildGlobal()
+}
+
+const NOOP=_=>null
+function buildContext(ctx0, option) {
+
+  const {
+    write=NOOP,
+    end=NOOP,
+    onError=NOOP,
+    masterPriv=null,
+    readAsBinary=false,
+  }=option
+
+  const ctx=buildGlobal()
+
+  const priv=masterPriv || {
+    isEnd: false,
+    echos: [],
+    syncs: [],
+    _defers: [],
+    __autoload_func: null,
+    __autoload_vars: {},
+  }
+
+  Object.assign(ctx, {
+    end,
+    onError,
+    isMaster: !masterPriv,
+  })
+
+  ctx.echo=(...x)=>{
+    if(priv.isEnd) return;
+    priv.echos.push(...x)
+  }
+
+  ctx.flush=_=>{
+    if(priv.isEnd) return;
+    const {echos}=priv
+    echos.splice(0).map(x=>write(toWritable(x, readAsBinary)))
+  }
+
+  ctx.var_dump=(...x)=>{
+    for(let o of x) {
+      ctx.echo(var_dump(o))
+    }
+  }
+
+  ctx.Sync={
+    Push: (...x)=>{
+      if(priv.isEnd) return;
+      priv.syncs.push(...x)
+    },
+  }
+
+  ctx.defer=fn=>{
+    if(priv.isEnd) return;
+    priv._defers.push(fn)
+  }
+
+  try{
+    ctx.require=_module.createRequire(ctx0.__filefullname)
+  }catch(e) {
+    ctx.require=require
+  }
+
+  ctx._exports={}
+  ctx.exports=o=>{
+    Object.assign(ctx._exports, o)
+  }
+
+  ctx.eval=code=>(new vm.Script(code)).runInNewContext(ctx)
+
+  ctx.include=(inc_filename, payload={})=>{
+    const inc=path.resolve(ctx0.__dirname, inc_filename)
+    const [_ctx0, _vm]=Compiler.compileSptcFile(inc)
+    const [_ctx, _]=buildContext({...payload, ..._ctx0}, {
+      write: ctx.write,
+      // end: ctx.end,
+      onError: ctx.onError,
+      masterPriv: priv,
+      readAsBinary,
+    })
+    const ret=executeVm(_vm, _ctx, priv)
+    return [_ctx._exports, x=>ret.constructor('return '+x)()]
+  }
+
+  ctx.__autoload=fn=>{
+    priv.__autoload_func=fn
+  }
+
+  let pctx=Object.assign({}, ctx0, ctx)
+  pctx=new Proxy(pctx, {
+    get: (target, prop, receiver)=>{
+      if(false===prop in target) {
+        if(prop in priv.__autoload_vars) {
+          return priv.__autoload_vars[prop]
+        }
+        if(priv.__autoload_func) {
+          let inc_fn=priv.__autoload_func(prop)
+          if(inc_fn) {
+            const [{default: d}, q]=ctx.include(inc_fn)
+            return priv.__autoload_vars[prop]=d || q(prop)
+          }
+        }
+      }
+      return target[prop]
+    },
+  })
+
+  return [pctx, priv]
+}
+
+function toWritable(x, asBinary=false) {
+  if(x && [Uint8Array, Buffer].includes(x.constructor)) {
+    return asBinary? x: Buffer.from(x).toString('utf8')
+  }
+  const str=(_=>{
+    if(typeof x==='string') return x
+    if(typeof x==='undefined') return 'undefined'
+    if(typeof x==='number') {
+      if(isNaN(x))return 'NaN'
+      if(x===Infinity) return 'Infinity'
+      if(x===-Infinity) return '-Infinity'
+    }
+    return JSON.stringify(x)+''
+  })()
+  return asBinary? Buffer.from(str): str
+}
+
+function executeVm(vm, ctx, priv) {
+  const {onError, end, flush, isMaster}=ctx
+  let ret
+  try{
+    ret=vm.runInNewContext(ctx)
+  }catch(e) {
+    onError(e)
+  }
+  ; (async _=>{
+
+    try{
+      const syncs=priv.syncs.length? Promise.all(priv.syncs): null
+      await Promise.resolve(syncs)
+      flush()
+      end()
+      if(isMaster) priv.isEnd=true
+    }catch(e) {
+      onError(e)
+    }
+
+    for(let fn of priv._defers.splice(0)) {
+      try{
+        fn()
+      }catch(e) {
+        onError(e)
+      }
+    }
+
+  })()
+  return ret
+}
+
+
+module.exports={
+  buildContext,
+  executeVm,
+  securityPatch,
+}

package/package.json

@@ -1,9 +1,16 @@
 {
   "name": "sptc",
-  "version": "0.0.0",
+  "version": "0.0.2",
   "description": "Simple Pretreat Toolkit CLI",
   "main": "index.js",
   "engines": {
     "node": ">=12"
+  },
+  "bin": {
+    "sptc": "./bin/sptc.js",
+    "sptcd": "./bin/sptcd.js"
+  },
+  "dependencies": {
+    "commander": "^10.0.1"
   }
 }

package/tests/bin.sh

@@ -0,0 +1,9 @@
+
+echo ============================
+echo ============================
+echo ============================
+echo ============================
+echo ============================
+echo Testing engine:
+
+node $0/../engine/test

package/tests/engine/TestModel.sjs

@@ -0,0 +1,7 @@
+<?js
+
+class TestModel{
+	static test() {
+	  return 'YYYY'+require(__dirname+'/m1').A()
+	}
+}

package/tests/engine/a.sjs

@@ -0,0 +1,58 @@
+<?js
+/*
+function test(key, fn) {
+  console.log('--> '+key+' :')
+  try{
+    const p=[GLOBAL, THIS, TOP].indexOf(o)
+    console.log(p<0? '-- pass --': ('  -->'+['GLOBAL', 'this', 'globalThis'][p]))
+  }catch(e) {
+    console.log('-- pass --')
+  }
+}
+
+
+test('nodeGlobal', _=>parseInt.constructor('return global')())
+test('nodeGlobalThis', _=>parseInt.constructor('return this')())
+test('realGlobal', _=>Function('return global')())
+test('realGlobalThis', _=>Function('return this')())
+test('controllableGlobal', _=>global)
+test('currentContext', _=>globalThis)
+test('currentThis', _=>this)
+
+*/
+
+
+#ifdef W1
+console.log('W1 defined!')
+process.exit()
+#endif
+
+
+console.log(_x)
+       function sleep(t) {
+         return new Promise(r=>setTimeout(r, t))
+       }
+
+__autoload(varname=>{
+	if(varname==='TestModel') return './TestModel.sjs'
+})
+
+const [v, _]=include('./ee.sjs', {P: 55})
+echo('['+v.ee+']')
+
+Sync.Push((async _=>{
+  echo(aa, 'LLL')
+  flush()
+  echo(1919)
+  var_dump({x: 3})
+  console.log(">>", eval+'')
+  defer(_=>{
+    console.log('defer')
+  })
+  flush()
+	await sleep(2e3)
+	echo('SLEEP=1e3')
+	setTimeout(_=>{
+	  echo(2929)
+	}, 2e3)
+})())

package/tests/engine/ee.sjs

@@ -0,0 +1,17 @@
+<?js
+exports({
+	ee: 444,
+})
+echo('---EE--', P*1e8)
+defer(_=>{
+	console.log("::EE<<")
+})
+
+echo('----------->', TestModel.test())
+
+Sync.Push(new Promise(r=>{
+	setTimeout(_=>{
+  	echo('xxxxxxxxx')
+  	r()
+	}, 2e3)
+}))

package/tests/engine/m1.js

@@ -0,0 +1,20 @@
+exports.A=_=>'m1 -> A'
+
+console.log('global.XXX - global.YYY', global.XXX, global.YYY)
+
+function test(key, fn) {
+	console.log('--> '+key+' :')
+	try{
+    console.log(fn())
+	}catch(e) {
+		console.log(' -- pass --')
+	}
+}
+
+test('nodeGlobal', _=>parseInt.constructor('return global')())
+test('nodeGlobalThis', _=>parseInt.constructor('return this')())
+test('realGlobal', _=>Function('return global')())
+test('realGlobalThis', _=>Function('return this')())
+test('controllableGlobal', _=>global)
+test('currentContext', _=>globalThis)
+test('currentThis', _=>this)

package/tests/engine/ppp.sjs

@@ -0,0 +1,6 @@
+<?js
+#ifdef XX
+echo('XX defined')
+#endif
+
+echo(333)

package/tests/engine/test-macro.js

@@ -0,0 +1,16 @@
+const {
+	executeSptcMacroFile,
+}=require('../../engine')
+
+console.log(executeSptcMacroFile('aa.js', {defs: ['XX']}, `
+#ifdef XX
+mkmk
+#def yy
+#endif
+
+#ifdef yy
+yy is defined
+#else
+yy is undefined
+#endif
+`))

package/tests/engine/test.js

@@ -0,0 +1,33 @@
+const {
+	executeSptcFile,
+	executeSptcFileEx,
+
+  securityPatch,
+}=require('../../engine')
+
+global.XXX=22
+
+securityPatch()
+
+global.YYY=22
+
+/*
+executeSptcFile(__dirname+'/a.sjs', {aa: 'AAA', GLOBAL: global, THIS: this, TOP: globalThis}, {
+	write: x=>process.stdout.write(x),
+	end: _=>console.log('>>end'),
+	onError: e=>console.log(e),
+})
+*/
+
+const option={
+	__DEV__: true,
+	macroOption: {
+		//defs: ['W1'],
+	},
+}
+executeSptcFileEx(__dirname+'/a.sjs', {aa: 'AAA', GLOBAL: global, THIS: this, TOP: globalThis}, option).then(out=>{
+	console.log(out, '<<END')
+}, e=>{
+	console.log(e)
+})
+

package/tests/www/index.sjs

@@ -0,0 +1,17 @@
+<?js
+echo($_REQUEST_FILE)
+echo('aaaa<pre>')
+var_dump($_GLOBAL, this)
+
+function sleep(t) {
+	return new Promise(r=>setTimeout(r, t))
+}
+
+Sync.Push((async _=>{
+	for(let i=0; i<10; i++) {
+  	echo(i*1e8+'<br/>')
+  	flush()
+  	await sleep(1e3)
+	}
+	echo('--done--')
+})())

package/utils/base.js

@@ -0,0 +1,65 @@
+
+const fs=require('fs')
+const crypto=require('crypto')
+const path=require('path')
+
+function readTextFile(filename) {
+  return fs.readFileSync(filename, 'utf8')
+}
+
+function mtime(filename) {
+  return fs.statSync(filename).mtime.getTime()
+}
+
+let _g=0
+function generate_uuid(prefix='') {
+  return [
+    prefix,
+    (_g=(_g+1)%1e8).toString(36),
+    (Date.now()+Math.random()).toString(36),
+  ].join('_')
+}
+
+function md5(str) {
+  return crypto.createHash('md5').update(str).digest('hex')
+}
+
+function safe_path(ref, dir) {
+  return path.normalize((ref? ref+'/': '')+path.normalize(dir))
+}
+
+function fileExists(fn) {
+  try{
+    return fs.statSync(fn).isFile()
+  }catch(e) {}
+}
+
+function getExtension(filename) {
+	return path.parse(filename).ext
+}
+
+function getLocalIpv4Addresses() {
+  const os=require('os')
+  let rr={'127.0.0.1': 1}
+  try{
+    const ii=os.networkInterfaces()
+    for(let a in ii) {
+      ii[a].map(x=>{
+        if(!x.family.match(/IPV4/i)) return;
+        rr[x.address]=1
+      })
+    }
+  }catch(e) {}
+  return Object.keys(rr)
+}
+
+module.exports={
+  readTextFile,
+  mtime,
+  md5,
+  safe_path,
+  fileExists,
+  generate_uuid,
+  getExtension,
+  getLocalIpv4Addresses,
+}