wao 0.7.2 → 0.8.2

package/esm/aoconnect.js

@@ -23,12 +23,12 @@ import { scheduler, mu, su, cu, acc } from "./test.js"
 import { is, clone, fromPairs, map, mergeLeft, isNil } from "ramda"
 import AR from "./tar.js"
 
-export const connect = (mem, { log = false, extensions = {} } = {}) => {
+export const connect = (mem, { cache, log = false, extensions = {} } = {}) => {
   const isMem = mem?.__type__ === "mem"
   if (!isMem) {
-    let args = {}
+    let args = { cache }
     if (mem?.SU_URL) {
-      args = mem
+      args = mergeLeft(mem, args)
       args.scheduler = scheduler
     }
     mem = new ArMem(args)
@@ -56,14 +56,14 @@ export const connect = (mem, { log = false, extensions = {} } = {}) => {
     return output
   }
 
-  const genMsg = (Id, p, data, Tags, from, Owner, dry = false) => {
+  const genMsg = async (Id, p, data, Tags, from, Owner, dry = false) => {
     if (!dry) p.height += 1
     return {
       Id,
       Target: p.id,
       Owner,
       Data: data?.length ? data : "",
-      "Block-Height": mem.height.toString(),
+      "Block-Height": (await mem.get("height")).toString(),
       Timestamp: Date.now().toString(),
       Module: p.module,
       From: from,
@@ -94,31 +94,10 @@ export const connect = (mem, { log = false, extensions = {} } = {}) => {
       },
     }
   }
-
   const spawn = async (opt = {}) => {
     if (!opt.module) throw Error("module missing")
     if (!opt.scheduler) throw Error("scheduler missing")
-    let mod = opt.module ?? mem.modules["aos2_0_1"]
-    let _module = null
-    const __dirname = await dirname()
-    let format = null
-    let wasm = mem.wasms[mod]?.data
-    if (!wasm) {
-      if (mem.wasms[mod]?.file) {
-        wasm = readFileSync(
-          resolve(__dirname, `lua/${mem.wasms[mod].file}.wasm`),
-        )
-        format = mem.wasms[mod].format
-      } else if (mem.txs[mod]) {
-        wasm = Buffer.from(mem.txs[mod].data, "base64")
-        format = tags(mem.txs[mod].tags)["Module-Format"]
-      }
-    } else {
-      format = mem.wasms[mod].format
-    }
-    format ??= "wasm64-unknown-emscripten-draft_2024_02_15"
-
-    if (!mod) throw Error("module not found")
+    const { mod, wasm, format } = await mem.getWasm(opt.module, mem)
     opt.tags = buildTags(
       null,
       mergeLeft(tags(opt.tags ?? []), {
@@ -132,7 +111,6 @@ export const connect = (mem, { log = false, extensions = {} } = {}) => {
       }),
     )
     let ex = false
-    opt.tags ??= []
     for (let v of opt.tags) if (v.name === "Type") ex = true
     if (!ex) opt.tags.push({ name: "Type", value: "Process" })
     const {
@@ -151,24 +129,28 @@ export const connect = (mem, { log = false, extensions = {} } = {}) => {
     await ar.postItems(item, su.jwk)
     const now = Date.now
     const t = tags(opt.tags)
-    const wdrive = extensions[t.Extension || "WeaveDrive"]
+    const ext = t.Extension || "WeaveDrive"
+    const wdrive = extensions[ext]
     let handle = null
     try {
       handle = await AoLoader(wasm, {
         format,
         WeaveDrive: wdrive,
         spawn: item,
-        module: mem.txs[mod],
+        module: await mem.get("txs", mod),
       })
     } catch (e) {}
     if (!handle) return null
+    let _module = null
     _module = { handle, id: mod }
     Date.now = now
-
     const _tags = tags(opt.tags)
     let res = null
     let memory = opt.memory ?? null
     let p = {
+      extension: ext,
+      item: item?.binary,
+      format,
       id: id,
       epochs: [],
       handle: _module.handle,
@@ -187,8 +169,8 @@ export const connect = (mem, { log = false, extensions = {} } = {}) => {
     } else if (_tags["On-Boot"] || true) {
       let data = ""
       if (_tags["On-Boot"] === "Data") data = opt.data ?? ""
-      else data = mem.msgs[_tags["On-Boot"]]?.data ?? ""
-      let msg = genMsg(id, p, data, opt.tags, owner, mu.addr, true)
+      else data = (await mem.get("msgs", _tags["On-Boot"]))?.data ?? ""
+      let msg = await genMsg(id, p, data, opt.tags, owner, mu.addr, true)
       const _env = genEnv({
         pid: p.id,
         owner: p.owner,
@@ -202,8 +184,7 @@ export const connect = (mem, { log = false, extensions = {} } = {}) => {
     } else {
       p.height += 1
     }
-    mem.msgs[id] = opt
-    mem.env[id] = p
+    await mem.set(opt, "msgs", id)
     if (_tags["Cron-Interval"]) {
       let [num, unit] = _tags["Cron-Interval"].split("-")
       let int = 0
@@ -236,9 +217,10 @@ export const connect = (mem, { log = false, extensions = {} } = {}) => {
           cronTags.push({ name: k.replace(/Cron-Tag-/, ""), value: _tags[k] })
         }
       }
-      mem.env[id].cronTags = cronTags
-      mem.env[id].span = int
+      p.cronTags = cronTags
+      p.span = int
     }
+    await mem.set(p, "env", id)
     return id
   }
 
@@ -252,9 +234,9 @@ export const connect = (mem, { log = false, extensions = {} } = {}) => {
   }
 
   const assign = async opt => {
-    const p = mem.env[opt.process]
+    const p = await mem.get("env", opt.process)
     if (!p) return null
-    let _opt = mem.msgs[opt.message]
+    let _opt = await mem.get("msgs", opt.message)
     let hash = genHashChain(p.hash, opt.message)
     p.hash = hash
     opt.tags = buildTags(
@@ -267,7 +249,7 @@ export const connect = (mem, { log = false, extensions = {} } = {}) => {
         Variant: "ao.TN.1",
         SDK: "aoconnect",
         Type: "Assignment",
-        "Block-Height": mem.height,
+        "Block-Height": await mem.get("height"),
         Process: opt.process,
         Message: opt.message,
         "Hash-Chain": hash,
@@ -305,22 +287,34 @@ export const connect = (mem, { log = false, extensions = {} } = {}) => {
         }
       }
       // check: is owner=mu.addr right?
-      const msg = genMsg(opt.message, p, data, _tags, from, mu.addr)
+      const msg = await genMsg(opt.message, p, data, _tags, from, mu.addr)
       const _env = genEnv({
         pid: p.id,
         owner: p.owner,
         module: p.module,
         auth: mu.addr,
       })
+      if (!p.handle) {
+        const { format, mod, wasm } = await mem.getWasm(p.modulea)
+        const wdrive = extensions[p.extention]
+        p.handle = await AoLoader(wasm, {
+          format,
+          WeaveDrive: wdrive,
+          spawn: new DataItem(p.item),
+          module: await mem.get("txs", mod),
+        })
+        mem.env[opt.process].handle = p.handle
+      }
       const res = await p.handle(p.memory, msg, _env)
       p.memory = res.Memory
       delete res.Memory
       p.res[opt.message] = res
       p.results.push(opt.message)
       p.txs.unshift({ id, ...opt })
-      mem.msgs[opt.message] = _opt
+      await mem.set(p, "env", opt.process)
+      await mem.set(_opt, "msgs", opt.message)
       for (const v of res.Messages ?? []) {
-        if (mem.env[v.Target]) {
+        if (await mem.get("env", v.Target)) {
           await message({
             process: v.Target,
             tags: v.Tags,
@@ -360,7 +354,7 @@ export const connect = (mem, { log = false, extensions = {} } = {}) => {
   }
 
   const message = async opt => {
-    const p = mem.env[opt.process]
+    const p = await mem.get("env", opt.process)
     if (!p) return null
     let ex = false
     let id = opt?.item?.id ?? ""
@@ -384,7 +378,7 @@ export const connect = (mem, { log = false, extensions = {} } = {}) => {
         target: opt.process,
       }))
     }
-    mem.msgs[id] = opt
+    await mem.set(opt, "msgs", id)
     await assign({
       message_item: item,
       message: id,
@@ -398,14 +392,14 @@ export const connect = (mem, { log = false, extensions = {} } = {}) => {
   return {
     message,
     unmonitor: async opt => {
-      const p = mem.env[opt.process]
+      const p = await mem.get("env", opt.process)
       try {
         clearInterval(p.cron)
         p.cron = null
       } catch (e) {}
     },
     monitor: async opt => {
-      const p = mem.env[opt.process]
+      const p = await mem.get("env", opt.process)
       if (isNil(p.cron)) {
         p.cron = setInterval(async () => {
           await message({
@@ -420,9 +414,9 @@ export const connect = (mem, { log = false, extensions = {} } = {}) => {
     spawn,
     assign,
     ar,
-    result: async opt => mem.env[opt.process].res[opt.message],
+    result: async opt => (await mem.get("env", opt.process)).res[opt.message],
     results: async opt => {
-      const p = mem.env[opt.process]
+      const p = await mem.get("env", opt.process)
       let results = []
       const limit = opt.limit ?? 25
       if (opt.sort === "DESC") {
@@ -439,7 +433,7 @@ export const connect = (mem, { log = false, extensions = {} } = {}) => {
       return { edges: results }
     },
     dryrun: async opt => {
-      const p = mem.env[opt.process]
+      const p = await mem.get("env", opt.process)
       if (!p) return null
       let id = opt.id ?? ""
       let owner = opt.owner ?? ""
@@ -447,7 +441,7 @@ export const connect = (mem, { log = false, extensions = {} } = {}) => {
         ;({ id, owner } = await ar.dataitem({ ...opt, target: opt.process }))
       }
       try {
-        const msg = genMsg(
+        const msg = await genMsg(
           id,
           p,
           opt.data ?? "",
@@ -470,6 +464,17 @@ export const connect = (mem, { log = false, extensions = {} } = {}) => {
             shared: memory.shared || false,
           })
         }
+        if (!p.handle) {
+          const { format, mod, wasm } = await mem.getWasm(p.modulea)
+          const wdrive = extensions[p.extention]
+          p.handle = await AoLoader(wasm, {
+            format,
+            WeaveDrive: wdrive,
+            spawn: new DataItem(p.item),
+            module: await mem.get("txs", mod),
+          })
+          mem.env[opt.process].handle = p.handle
+        }
         const res = await p.handle(p.memory, msg, _env)
         return res
       } catch (e) {

package/esm/armem.js

@@ -1,12 +1,83 @@
 import Arweave from "arweave"
 import { last } from "ramda"
-import { buildTags } from "./utils.js"
+import { tags, buildTags, dirname } from "./utils.js"
+import { open } from "lmdb"
+import { readFileSync } from "fs"
+import { resolve } from "path"
+
 export default class ArMem {
-  constructor({ MU_URL, CU_URL, SU_URL, GATEWAY_URL, scheduler } = {}) {
+  constructor({ MU_URL, CU_URL, SU_URL, GATEWAY_URL, scheduler, cache } = {}) {
     this.__type__ = "mem"
+    this.isInit = false
+    if (cache) this.db = open({ path: cache, compression: true })
     this.arweave = Arweave.init()
     this.arweave.transactions.getTransactionAnchor = () => this.getAnchor()
     this.arweave.transactions.getPrice = () => 0
+    this.scheduler = scheduler
+    this.SU_URL = SU_URL
+    this.initSync()
+  }
+  async putAll(key) {
+    for (let k in this[key]) await this.set(this[key][k], key, k)
+  }
+  async init() {
+    if (this.isInit) return
+    this.isInit = true
+    if (this.db) {
+      for (const v of ["height", "blocks"]) this[v] = await this.get(v)
+      for (const v of [
+        "txs",
+        "jwks",
+        "env",
+        "modules",
+        "wasms",
+        "addrmap",
+        "blockmap",
+        "modmap",
+        "msgs",
+      ]) {
+        this[v] ??= {}
+        const items = await this.db.getKeys({ start: v, end: v + "a" })
+        for (const v2 of items) {
+          const key = v2.split(".")[0]
+          const field = v2.split(".")[1]
+          if (key === v) this[v][field] = await this.db.get(v2)
+        }
+      }
+    } else {
+      for (const v of ["height", "blocks"]) await this.set(this[v], v)
+      for (const v of ["modules", "wasms", "addrmap", "blockmap"]) {
+        await this.putAll(v)
+      }
+    }
+  }
+  async getWasm(module) {
+    let mod = module ?? this.modules.aos2_0_1
+    if (!mod) throw Error("module not found")
+    const __dirname = await dirname()
+    let format = null
+    let _wasm = await this.wasms[mod]
+    let wasm = _wasm?.data
+    if (!wasm) {
+      if (_wasm?.file) {
+        wasm = readFileSync(
+          resolve(__dirname, `lua/${this.wasms[mod].file}.wasm`),
+        )
+        format = _wasm.format
+      } else {
+        const tx = await this.get("txs", mod)
+        if (tx) {
+          wasm = Buffer.from(tx.data, "base64")
+          format = tags(tx.tags)["Module-Format"]
+        }
+      }
+    } else {
+      format = _wasm.format
+    }
+    format ??= "wasm64-unknown-emscripten-draft_2024_02_15"
+    return { format, mod, wasm }
+  }
+  initSync() {
     this.addrmap = {}
     this.txs = {}
     this.jwks = {}
@@ -54,19 +125,19 @@ export default class ArMem {
         }),
       }
     }
-    if (scheduler && SU_URL) {
-      const key = scheduler
+    if (this.scheduler && this.SU_URL) {
+      const key = this.scheduler
       txs.push(key)
-      this.addrmap[scheduler] = { address: scheduler }
+      this.addrmap[this.scheduler] = { address: this.scheduler }
       this.txs[key] = {
         id: key,
         block: 0,
-        owner: scheduler,
+        owner: this.scheduler,
         tags: buildTags(null, {
           "Data-Protocol": "ao",
           Variant: "ao.TN.1",
           Type: "Scheduler-Location",
-          Url: SU_URL,
+          Url: this.SU_URL,
           "Time-To-Live": 1000 * 60 * 60 * 24 * 365 * 10,
         }),
       }
@@ -86,4 +157,20 @@ export default class ArMem {
       ? "Do_Uc2Sju_ffp6Ev0AnLVdPtot15rvMjP-a9VVaA5fM"
       : last(this.blockmap[last(this.blocks)].txs)
   }
+  async get(key, field) {
+    await this.init()
+    if (!field) return this[key]
+    return this[key]?.[field]
+  }
+  async set(val, key, field) {
+    await this.init()
+    if (!field) {
+      this[key] = val
+      if (this.db) await this.db.put(`${key}`, this[key])
+    } else {
+      this[key] ??= {}
+      this[key][field] = val
+      if (this.db) await this.db.put(`${key}.${field}`, this[key][field])
+    }
+  }
 }

package/esm/cli.js

@@ -0,0 +1,39 @@
+#!/usr/bin/env node
+import pm2 from "pm2"
+import { dirname } from "./utils.js"
+import { resolve } from "path"
+const args = process.argv.slice(2)
+
+pm2.connect(false, async err => {
+  if (err) {
+    console.error("Error connecting to PM2:", err)
+    process.exit(2)
+  }
+
+  pm2.start(
+    {
+      script: resolve(await dirname(), "run.js"),
+      nodeArgs: "--experimental-wasm-memory64",
+      instances: 1,
+      force: true,
+      args: args,
+      daemon: false,
+      name: "wao",
+    },
+    err => {
+      if (err) {
+        console.error("Error starting process:", err)
+        pm2.disconnect()
+        process.exit(2)
+      }
+    },
+  )
+  pm2.streamLogs("all", 0, false)
+})
+
+process.on("SIGINT", () => {
+  pm2.delete("wao", err => {
+    pm2.disconnect()
+    process.exit(err ? 1 : 0)
+  })
+})

package/esm/run.js

@@ -1,9 +1,20 @@
 #!/usr/bin/env -S node --experimental-wasm-memory64
 import yargs from "yargs"
-let { port = 4000 } = yargs(process.argv.slice(2)).argv
-
+import { resolve } from "path"
+import { unlinkSync } from "fs"
 import Server from "./server.js"
-
-const main = async () => new Server({ log: true, port })
+let {
+  reset = false,
+  memory = false,
+  port = 4000,
+  db = ".cache",
+} = yargs(process.argv.slice(2)).argv
+db = memory ? null : resolve(process.cwd(), db)
+if (reset) {
+  try {
+    unlinkSync(db)
+  } catch (e) {}
+}
+const main = async () => new Server({ log: true, port, db })
 
 main()

package/esm/server.js

@@ -16,6 +16,7 @@ class Server {
     cu = 4004,
     aoconnect,
     log = false,
+    db,
     port,
   } = {}) {
     if (port) {
@@ -35,7 +36,7 @@ class Server {
       mem,
       monitor,
       unmonitor,
-    } = connect(aoconnect, { log })
+    } = connect(aoconnect, { log, cache: db })
     this.monitor = monitor
     this.unmonitor = unmonitor
     this.spawn = spawn
@@ -288,7 +289,7 @@ class Server {
         v.close(() => {
           count += 1
           if (count >= 4) {
-            console.log("servers closed!", count)
+            console.log("servers closed!")
             res()
           }
         })

package/esm/tao.js

@@ -64,7 +64,11 @@ class AO extends MAO {
       monitor,
       unmonitor,
       mem,
-    } = connect(opt.mem, { extensions: opt.extensions })
+    } = connect(opt.mem, {
+      extensions: opt.extensions,
+      cache: opt.cache,
+      reset: opt.reset,
+    })
     this.module = mem.modules.aos2_0_1
     this.assign = assign
     this.result = async (...opt) => {

package/esm/tar.js

@@ -8,6 +8,7 @@ import base64url from "base64url"
 import ArMem from "./armem.js"
 import GQL from "./tgql.js"
 import { last, is, includes, isNil } from "ramda"
+
 class AR extends MAR {
   constructor(opt = {}) {
     super({ ...opt, in_memory: true })
@@ -60,11 +61,8 @@ class AR extends MAR {
       for (const t of di.tags)
         if (t.name === "Content-Type") data_type = t.value
       const owner = await this.owner(di)
-      this.mem.addrmap[owner] = { key: di.owner, address: owner }
+      await this.mem.set({ key: di.owner, address: owner }, "addrmap", owner)
       let data = di.data
-      try {
-        //data = base64url.decode(di.data)
-      } catch (e) {}
       let _item = {
         _data: { size: data_size, type: data_type },
         anchor: di.anchor,
@@ -76,7 +74,7 @@ class AR extends MAR {
         tags: di.tags,
         data,
       }
-      this.mem.txs[await di.id] = _item
+      await this.mem.set(_item, "txs", await di.id)
       _items.push(_item)
     }
     const bundle = await bundleAndSignData(items, new ArweaveSigner(jwk))
@@ -96,21 +94,22 @@ class AR extends MAR {
 
     let res = null
     if (!tx.id) await this.mem.arweave.transactions.sign(tx, jwk)
-    this.mem.height += 1
+    let height = (await this.mem.get("height")) + 1
+    await this.mem.set(height, "height")
     let block = {
       id: tx.id,
       timestamp: Date.now(),
-      height: this.mem.height,
-      previous: last(this.mem.blocks) ?? "",
+      height,
+      previous: last(await this.mem.get("blocks")) ?? "",
       txs: [],
     }
     let msg = null
     if (items) {
       for (const item of items) {
-        this.mem.txs[item.id] = item
-        this.mem.txs[item.id].parent = { id: tx.id }
-        this.mem.txs[item.id].bundledIn = { id: tx.id }
-        this.mem.txs[item.id].anchor = ""
+        let _txs = item
+        _txs.parent = { id: tx.id }
+        _txs.bundledIn = { id: tx.id }
+        _txs.anchor = ""
         const _tags = t(item.tags)
         if (
           includes(_tags.Type, [
@@ -129,9 +128,9 @@ class AR extends MAR {
         for (const v of item.item.tags) {
           if (v.name === "Content-Type") data_type = v.value
         }
-        //this.mem.txs[tx.id]._data = { size: tx.data_size, type: data_type }
         block.txs.push(item.id)
-        this.mem.txs[item.id].block = block.id
+        _txs.block = block.id
+        await this.mem.set(_txs, "txs", item.id)
       }
     }
     let _tags = []
@@ -143,22 +142,29 @@ class AR extends MAR {
     }
     const __tags = t(_tags)
     if (__tags.Type === "Module") {
-      this.mem.wasms[tx.id] = {
-        data: Buffer.from(tx.data, "base64"),
-        format: __tags["Module-Format"],
-      }
+      await this.mem.set(
+        {
+          data: Buffer.from(tx.data, "base64"),
+          format: __tags["Module-Format"],
+        },
+        "wasms",
+        tx.id,
+      )
     }
     tx.tags = _tags
     tx.owner = await this.arweave.wallets.jwkToAddress({ n: tx.owner })
-    this.mem.txs[tx.id] = tx
+    let _txs = tx
     block.txs.push(tx.id)
-    this.mem.txs[tx.id].block = block.id
-    this.mem.blocks.push(block.id)
-    this.mem.blockmap[block.id] = block
+    _txs.block = block.id
+    await this.mem.set(_txs, "txs", tx.id)
+    let blocks = await this.mem.get("blocks")
+    blocks.push(block.id)
+    await this.mem.set(blocks, "blocks")
+    await this.mem.set(block, "blockmap", block.id)
 
     if (jwk) {
       const owner = await this.arweave.wallets.jwkToAddress(jwk)
-      this.mem.addrmap[owner] = { address: owner, key: jwk.n }
+      await this.mem.set({ address: owner, key: jwk.n }, "addrmap", owner)
     }
     res = { id: tx.id, status: 200, statusText: "200" }
     if (this.log) {
@@ -174,7 +180,7 @@ class AR extends MAR {
   }
 
   async tx(id) {
-    return this.mem.txs[id]
+    return await this.mem.get("txs", id)
   }
 
   async data(id, _string, log) {
@@ -184,7 +190,7 @@ class AR extends MAR {
       if (!isNil(_string.decode)) decode = _string.decode
       if (!isNil(_string.string)) string = _string.string
     }
-    let tx = this.mem.txs[id]
+    let tx = await this.mem.get("txs", id)
     let _data = tx?.data ?? null
     if (tx?.format === 2 && _data) {
       _data = Buffer.from(_data, "base64")

package/package.json

@@ -1,7 +1,10 @@
 {
   "name": "wao",
-  "version": "0.7.2",
-  "description": "",
+  "version": "0.8.2",
+  "bin": {
+    "wao": "./cjs/cli.js",
+    "wao-esm": "./esm/cli.js"
+  },
   "main": "cjs/index.js",
   "module": "esm/index.js",
   "exports": {
@@ -18,8 +21,6 @@
       "import": "./esm/test.js"
     }
   },
-  "author": "",
-  "license": "MIT",
   "dependencies": {
     "@babel/plugin-transform-modules-commonjs": "^7.24.8",
     "@permaweb/ao-loader": "^0.0.44",
@@ -31,19 +32,15 @@
     "cors": "^2.8.5",
     "express": "^4.21.2",
     "graphql": "^16.10.0",
+    "lmdb": "^3.2.2",
+    "pm2": "^5.4.3",
     "ramda": "^0.30.1",
     "yargs": "^17.7.2"
   },
   "devDependencies": {
     "@babel/cli": "^7.24.8",
     "@babel/core": "^7.25.2",
-    "@babel/preset-env": "^7.25.3",
-    "chai": "^5.1.1",
-    "mocha": "^10.7.3"
-  },
-  "bin": {
-    "wao": "./cjs/run.js",
-    "wao-esm": "./esm/run.js"
+    "@babel/preset-env": "^7.25.3"
   },
   "scripts": {
     "server": "node --experimental-wasm-memory64 cjs/run.js"