wao 0.6.0 → 0.6.2

package/esm/ao.js

@@ -46,6 +46,7 @@ import {
   mergeOut,
   isOutComplete,
   jsonToStr,
+  optAO,
 } from "./utils.js"
 
 function createDataItemSigner2(wallet) {
@@ -61,15 +62,26 @@ function createDataItemSigner2(wallet) {
 }
 
 class AO {
-  constructor({
-    authority = srcs.authority,
-    module,
-    module_type = "aos2",
-    scheduler = srcs.scheduler,
-    aoconnect,
-    ar = {},
-    in_memory = false,
-  } = {}) {
+  constructor(opt = {}) {
+    let _port = null
+    if (typeof opt === "number") {
+      _port = opt
+      opt = {}
+    }
+    let {
+      authority = srcs.authority,
+      module,
+      module_type = "aos2",
+      scheduler = srcs.scheduler,
+      aoconnect,
+      ar,
+      in_memory = false,
+      port,
+    } = opt
+
+    if (!_port && port) _port = port
+    if (!aoconnect && _port) aoconnect = optAO(_port)
+    if (!ar && _port) ar = { port: _port }
     if (!module) {
       switch (module_type) {
         case "sqlite":
@@ -83,14 +95,13 @@ class AO {
       }
     }
     this.__type__ = "ao"
-
     if (ar?.__type__ === "ar") this.ar = ar
     else {
       let _ar = typeof ar === "object" ? ar : {}
-      this.ar = new AR(ar)
+      this.ar = new AR(_ar)
     }
-    if (in_memory) {
-    } else if (aoconnect) {
+
+    if (!in_memory && aoconnect) {
       const {
         results,
         assign,
@@ -620,6 +631,7 @@ class AO {
     while (attempts > 0) {
       if (!this.in_memory) await wait(1000)
       const { res, err: _err } = await this.dry({ pid, data: "#Inbox" })
+      if (res?.error) return { err: res.error, pid }
       if (typeof res?.Output === "object") break
       attempts -= 1
       if (attempts === 0) err = "timeout"

package/esm/aoconnect.js

@@ -23,7 +23,7 @@ import { scheduler, mu, su, cu, acc } from "./test.js"
 import { is, clone, fromPairs, map, mergeLeft, isNil } from "ramda"
 import AR from "./tar.js"
 
-export const connect = (mem, log = false) => {
+export const connect = (mem, { log = false, extensions = {} }) => {
   const isMem = mem?.__type__ === "mem"
   if (!isMem) {
     let args = {}
@@ -34,7 +34,8 @@ export const connect = (mem, log = false) => {
     mem = new ArMem(args)
   }
   const ar = new AR({ mem, log })
-  const WeaveDrive = new weavedrive(ar).drive
+  for (const k in extensions) extensions[k] = new extensions[k](ar).ext
+  extensions.WeaveDrive = new weavedrive(ar).ext
 
   const transform = input => {
     const output = { Tags: [] }
@@ -149,9 +150,10 @@ export const connect = (mem, log = false) => {
     if (opt.item) opt.data = base64url.decode(item.data)
     await ar.postItems(item, su.jwk)
     const now = Date.now
+    const t = tags(opt.tags)
     const handle = await AoLoader(wasm, {
       format,
-      WeaveDrive,
+      WeaveDrive: extensions[t.Extension],
       spawn: item,
       module: mem.txs[mod],
       blockHeight: "100",
@@ -378,7 +380,6 @@ export const connect = (mem, log = false) => {
         target: opt.process,
       }))
     }
-    //await ar.postItems(item, su.jwk)
     mem.msgs[id] = opt
     await assign({
       message_item: item,
@@ -458,11 +459,11 @@ export const connect = (mem, log = false) => {
           auth: mu.addr,
         })
         function cloneMemory(memory) {
-          const buffer = memory.buffer.slice(0) // Clone the ArrayBuffer
+          const buffer = memory.buffer.slice(0)
           return new WebAssembly.Memory({
-            initial: memory.buffer.byteLength / 65536, // Memory size in pages (64KB per page)
+            initial: memory.buffer.byteLength / 65536,
             maximum: memory.maximum || undefined,
-            shared: memory.shared || false, // Retain sharing if applicable
+            shared: memory.shared || false,
           })
         }
         const res = await p.handle(p.memory, msg, _env)

package/esm/helpers.js

@@ -3,6 +3,7 @@ import assert from "assert"
 import { createDataItemSigner, connect } from "@permaweb/aoconnect"
 import { dirname as _dirname, resolve } from "path"
 import { mkdirSync, existsSync, writeFileSync, readFileSync } from "fs"
+import { optAO } from "./utils.js"
 import yargs from "yargs"
 
 let {
@@ -45,12 +46,7 @@ export class Testnet {
   constructor({ port = 4000, arweave, aoconnect, docker = false } = {}) {
     this.docker = docker
     this.arweave = arweave ?? { port }
-    this.aoconnect = aoconnect ?? {
-      MU_URL: `http://localhost:${port + 2}`,
-      SU_URL: `http://localhost:${port + 3}`,
-      CU_URL: `http://localhost:${port + 4}`,
-      GATEWAY_URL: `http://localhost:${port}`,
-    }
+    this.aoconnect = aoconnect ?? optAO(port)
     this.ar = new AR(this.arweave)
   }
   async init(jwk) {
@@ -60,7 +56,7 @@ export class Testnet {
     this.src = await new Src({ ar: this.ar }).init()
     await this.ar.init(jwk)
     this.jwk = this.ar.jwk
-    this.addr = this.ar.a
+    this.addr = this.ar.addr
     this.gql = this.ar.gql
     this.module_src = await this.src.upload("aos2_0_1", "wasm")
     this.ao = await new AO({
@@ -119,11 +115,7 @@ export const setup = async ({
 
   // ar
   arweave ??= { port: 4000 }
-  aoconnect ??= {
-    MU_URL: "http://localhost:4002",
-    CU_URL: "http://localhost:4004",
-    GATEWAY_URL: "http://localhost:4000",
-  }
+  aoconnect ??= optAO(4000)
   const ar = new AR(arweave)
   await ar.gen("10")
   const src = new Src({ ar, dir })

package/esm/lua/weavedb_mock.lua

@@ -0,0 +1,43 @@
+local json = require("json")
+function rollup (id)
+  local file = io.open("/rollup/" .. id)
+  local data = nil
+  if file then data = file:read(file:seek('end')) end
+  file:close()
+  return data
+end
+
+function get (col, doc)
+  local file = io.open("/data/" .. col .. "/" .. doc)
+  local data = nil
+  if file then data = file:read(file:seek('end')) end
+  file:close()
+  return data
+end
+
+Handlers.add(
+  "Rollup",
+  "Rollup",
+  function (msg)
+    msg.reply({ Data = "committed!" })
+  end
+)
+
+Handlers.add(
+  "Finalize",
+  "Finalize",
+  function (msg)
+    local data = json.decode(rollup(msg.TXID))
+    msg.reply({ Data = "finalized!" })
+  end
+)
+
+
+Handlers.add(
+  "Get",
+  "Get",
+  function (msg)
+    msg.reply({ Data = get(msg.col, msg.doc) })
+  end
+)
+

package/esm/server.js

@@ -27,7 +27,7 @@ class Server {
       mem,
       monitor,
       unmonitor,
-    } = connect(aoconnect, log)
+    } = connect(aoconnect, { log })
     this.monitor = monitor
     this.unmonitor = unmonitor
     this.spawn = spawn

package/esm/tao.js

@@ -64,7 +64,7 @@ class AO extends MAO {
       monitor,
       unmonitor,
       mem,
-    } = connect(opt.mem)
+    } = connect(opt.mem, { extensions: opt.extensions })
     this.module = mem.modules.aos2_0_1
     this.assign = assign
     this.result = async (...opt) => {

package/esm/utils.js

@@ -519,7 +519,21 @@ const toGraphObj = ({ query, variables }) => {
   }
   return { tar, args }
 }
+const optAO = port => {
+  return {
+    MU_URL: `http://localhost:${port + 2}`,
+    SU_URL: `http://localhost:${port + 3}`,
+    CU_URL: `http://localhost:${port + 4}`,
+    GATEWAY_URL: `http://localhost:${port}`,
+  }
+}
+const optServer = port => {
+  return { ar: port, mu: port + 2, su: port + 3, cu: port + 4 }
+}
+
 export {
+  optAO,
+  optServer,
   toGraphObj,
   jsonToStr,
   mergeChecks,

package/esm/weavedb.js

@@ -0,0 +1,309 @@
+import { resolve } from "path"
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"
+const KB = 1024
+const MB = KB * 1024
+const CACHE_SZ = 32 * KB
+const CHUNK_SZ = 128 * MB
+const NOTIFY_SZ = 512 * MB
+const log = console.log
+const rand = Math.floor(Math.random() * 1000000).toString()
+
+export default class WeaveDB {
+  constructor(ar, dir) {
+    dir ??= resolve(import.meta.dirname, ".db")
+    const kv_dir = resolve(dir, rand)
+    const data_dir = resolve(kv_dir, "data")
+    const rollup_dir = resolve(kv_dir, "rollup")
+    for (const v of [dir, kv_dir, rollup_dir, data_dir]) {
+      if (!existsSync(v)) mkdirSync(v)
+    }
+    this.ext = (mod, FS) => {
+      let cache = { data: {}, rollup: {} }
+      return {
+        async create(id) {
+          let properties = { isDevice: false, contents: null }
+          if (!FS.analyzePath("/rollup/").exists) FS.mkdir("/rollup/")
+          let node = FS.createFile("/", "rollup/" + id, properties, true, false)
+
+          // Set initial parame
+          let data = await ar.data(id)
+          const bytesLength = data?.length ?? 0
+          node.total_size = Number(bytesLength)
+          node.cache = new Uint8Array(0)
+          node.position = 0
+
+          // Add a function that defers querying the file size until it is asked the first time.
+          Object.defineProperties(node, {
+            usedBytes: { get: () => bytesLength },
+          })
+
+          // Now we have created the file in the emscripten FS, we can open it as a stream
+          let stream = FS.open("/rollup/" + id, "r")
+
+          //console.log("JS: Created file: ", id, " fd: ", stream.fd);
+          return stream
+        },
+        async createData(col, doc, val) {
+          let properties = { isDevice: false, contents: null }
+          if (!FS.analyzePath("/data/").exists) FS.mkdir("/data/")
+          if (!FS.analyzePath(`/data/${col}`).exists) FS.mkdir(`/data/${col}`)
+          let node = FS.createFile(
+            "/",
+            `data/${col}/${doc}`,
+            properties,
+            true,
+            false,
+          )
+
+          // Set initial parame
+          let data = val
+          if (!val) {
+            const col_dir = resolve(data_dir, col)
+            const _data =
+              readFileSync(resolve(col_dir, `${doc}.json`), "utf8") ?? ""
+            data = Buffer.from(_data, "utf8")
+          }
+          const bytesLength = data?.length ?? 0
+          node.total_size = Number(bytesLength)
+          node.cache = new Uint8Array(0)
+          node.position = 0
+
+          // Add a function that defers querying the file size until it is asked the first time.
+          Object.defineProperties(node, {
+            usedBytes: { get: () => bytesLength },
+          })
+
+          // Now we have created the file in the emscripten FS, we can open it as a stream
+          let stream = FS.open("/data/" + `${col}/${doc}`, "r")
+
+          //console.log("JS: Created file: ", id, " fd: ", stream.fd);
+          return stream
+        },
+        async open(filename, val) {
+          const pathCategory = filename.split("/")[1]
+          if (pathCategory === "rollup") {
+            //log("JS: Opening ID: ", id)
+            const id = filename.split("/")[2]
+            if (FS.analyzePath(filename).exists) {
+              let stream = FS.open(filename, "r")
+              if (stream.fd) return stream.fd
+              console.log("JS: File not found: ", filename)
+              return 0
+            } else {
+              //console.log("JS: Open => Creating file: ", id);
+              const stream = await this.create(id)
+              //console.log("JS: Open => Created file: ", id, " fd: ", stream.fd);
+              return stream.fd
+            }
+          } else if (pathCategory === "data") {
+            //log("JS: Opening ID: ", id)
+            const col = filename.split("/")[2]
+            const doc = filename.split("/")[3]
+            if (FS.analyzePath(filename).exists) {
+              let stream = FS.open(filename, "r")
+              if (stream.fd) return stream.fd
+              console.log("JS: File not found: ", filename)
+              return 0
+            } else {
+              //console.log("JS: Open => Creating file: ", id);
+              const stream = await this.createData(col, doc, val)
+              //console.log("JS: Open => Created file: ", id, " fd: ", stream.fd);
+              return stream.fd
+            }
+          } else {
+            console.log("JS: Invalid path category: ", pathCategory)
+            return 0
+          }
+        },
+        async read(fd, raw_dst_ptr, raw_length, val) {
+          let to_read = Number(raw_length)
+          let dst_ptr = Number(raw_dst_ptr)
+          let stream = 0
+          for (let i = 0; i < FS.streams.length; i++) {
+            if (FS.streams[i].fd === fd) stream = FS.streams[i]
+          }
+          // Satisfy what we can with the cache first
+          let bytes_read = this.readFromCache(stream, dst_ptr, to_read)
+          stream.position += bytes_read
+          stream.lastReadPosition = stream.position
+          dst_ptr += bytes_read
+          to_read -= bytes_read
+
+          // Return if we have satisfied the request
+
+          //console.log("KV: Satisfied request with cache. Returning...")
+          if (to_read === 0) return bytes_read
+
+          //console.log("KV: Read from cache: ", bytes_read, " Remaining to read: ", to_read)
+
+          const chunk_download_sz = Math.max(to_read, CACHE_SZ)
+          const to = Math.min(
+            stream.node.total_size,
+            stream.position + chunk_download_sz,
+          )
+          let data = val
+          if (!data) {
+            const sp = stream.path.split("/")
+            if (sp[1] === "data") {
+              const col_dir = resolve(data_dir, sp[2])
+              const _data =
+                readFileSync(resolve(col_dir, `${sp[3]}.json`), "utf8") ?? ""
+              data = Buffer.from(_data, "utf8")
+            } else {
+              data = await ar.data(stream.node.name)
+              try {
+                const json = JSON.parse(Buffer.from(data).toString())
+                for (const v of json.diffs) {
+                  const val = Buffer.from(JSON.stringify(v.data))
+                  const _fd = await this.open(
+                    `/data/${v.collection}/${v.doc}`,
+                    val,
+                  )
+                  const col_dir = resolve(data_dir, v.collection)
+                  if (!existsSync(col_dir)) mkdirSync(col_dir)
+                  writeFileSync(
+                    resolve(col_dir, `${v.doc}.json`),
+                    JSON.stringify(v.data),
+                  )
+                  await this.read(_fd, _fd, val.length, val)
+                }
+              } catch (e) {
+                log(e)
+              }
+            }
+          }
+          // Extract the Range header to determine the start and end of the requested chunk
+          const start = 0
+          const end = data.length
+          // Create a ReadableStream for the requested chunk
+          const chunk = data.subarray(start, end)
+          const response = new Response(
+            new ReadableStream({
+              start(controller) {
+                controller.enqueue(chunk) // Push the chunk to the stream
+                controller.close() // Close the stream when done
+              },
+            }),
+            {
+              headers: { "Content-Length": chunk.length.toString() },
+            },
+          )
+          const reader = response.body.getReader()
+          let bytes_until_cache = CHUNK_SZ
+          let bytes_until_notify = NOTIFY_SZ
+          let downloaded_bytes = 0
+          let cache_chunks = []
+
+          try {
+            while (true) {
+              const { done, value: chunk_bytes } = await reader.read()
+              if (done) break
+              // Update the number of downloaded bytes to be _all_, not just the write length
+              downloaded_bytes += chunk_bytes.length
+              bytes_until_cache -= chunk_bytes.length
+              bytes_until_notify -= chunk_bytes.length
+
+              // Write bytes from the chunk and update the pointer if necessary
+              const write_length = Math.min(chunk_bytes.length, to_read)
+              if (write_length > 0) {
+                //console.log("KV: Writing: ", write_length, " bytes to: ", dst_ptr)
+                mod.HEAP8.set(chunk_bytes.subarray(0, write_length), dst_ptr)
+                dst_ptr += write_length
+                bytes_read += write_length
+                stream.position += write_length
+                to_read -= write_length
+              }
+
+              if (to_read == 0) {
+                // Add excess bytes to our cache
+                const chunk_to_cache = chunk_bytes.subarray(write_length)
+                //console.log("KV: Cacheing excess: ", chunk_to_cache.length)
+                cache_chunks.push(chunk_to_cache)
+              }
+              if (bytes_until_cache <= 0) {
+                console.log("KV: Chunk size reached. Compressing cache...")
+                stream.node.cache = this.addChunksToCache(
+                  stream.node.cache,
+                  cache_chunks,
+                )
+                cache_chunks = []
+                bytes_until_cache = CHUNK_SZ
+              }
+
+              if (bytes_until_notify <= 0) {
+                console.log(
+                  "KV: Downloaded: ",
+                  (downloaded_bytes / stream.node.total_size) * 100,
+                  "%",
+                )
+                bytes_until_notify = NOTIFY_SZ
+              }
+            }
+          } catch (error) {
+            console.error("KV: Error reading the stream: ", error)
+          } finally {
+            reader.releaseLock()
+          }
+          // If we have no cache, or we have not satisfied the full request, we need to download the rest
+          // Rebuild the cache from the new cache chunks
+          stream.node.cache = this.addChunksToCache(
+            stream.node.cache,
+            cache_chunks,
+          )
+
+          // Update the last read position
+          stream.lastReadPosition = stream.position
+          return bytes_read
+        },
+        close(fd) {
+          let stream = 0
+          for (let i = 0; i < FS.streams.length; i++) {
+            if (FS.streams[i].fd === fd) stream = FS.streams[i]
+          }
+          FS.close(stream)
+        },
+
+        readFromCache(stream, dst_ptr, length) {
+          // Check if the cache has been invalidated by a seek
+          if (stream.lastReadPosition !== stream.position) {
+            //console.log("KV: Invalidating cache for fd: ", stream.fd, " Current pos: ", stream.position, " Last read pos: ", stream.lastReadPosition)
+            stream.node.cache = new Uint8Array(0)
+            return 0
+          }
+          // Calculate the bytes of the request that can be satisfied with the cache
+          let cache_part_length = Math.min(length, stream.node.cache.length)
+          let cache_part = stream.node.cache.subarray(0, cache_part_length)
+          mod.HEAP8.set(cache_part, dst_ptr)
+          // Set the new cache to the remainder of the unused cache and update pointers
+          stream.node.cache = stream.node.cache.subarray(cache_part_length)
+
+          return cache_part_length
+        },
+
+        addChunksToCache(old_cache, chunks) {
+          // Make a new cache array of the old cache length + the sum of the chunk lengths, capped by the max cache size
+          let new_cache_length = Math.min(
+            old_cache.length +
+              chunks.reduce((acc, chunk) => acc + chunk.length, 0),
+            CACHE_SZ,
+          )
+          let new_cache = new Uint8Array(new_cache_length)
+          // Copy the old cache to the new cache
+          new_cache.set(old_cache, 0)
+          // Load the cache chunks into the new cache
+          let current_offset = old_cache.length
+          for (let chunk of chunks) {
+            if (current_offset < new_cache_length) {
+              new_cache.set(
+                chunk.subarray(0, new_cache_length - current_offset),
+                current_offset,
+              )
+              current_offset += chunk.length
+            }
+          }
+          return new_cache
+        },
+      }
+    }
+  }
+}