wao 0.6.0 → 0.6.1

package/esm/ao.js

@@ -46,6 +46,7 @@ import {
   mergeOut,
   isOutComplete,
   jsonToStr,
+  optAO,
 } from "./utils.js"
 
 function createDataItemSigner2(wallet) {
@@ -61,15 +62,26 @@ function createDataItemSigner2(wallet) {
 }
 
 class AO {
-  constructor({
-    authority = srcs.authority,
-    module,
-    module_type = "aos2",
-    scheduler = srcs.scheduler,
-    aoconnect,
-    ar = {},
-    in_memory = false,
-  } = {}) {
+  constructor(opt = {}) {
+    let _port = null
+    if (typeof opt === "number") {
+      _port = opt
+      opt = {}
+    }
+    let {
+      authority = srcs.authority,
+      module,
+      module_type = "aos2",
+      scheduler = srcs.scheduler,
+      aoconnect,
+      ar,
+      in_memory = false,
+      port,
+    } = opt
+
+    if (!_port && port) _port = port
+    if (!aoconnect && _port) aoconnect = optAO(_port)
+    if (!ar && _port) ar = { port: _port }
     if (!module) {
       switch (module_type) {
         case "sqlite":
@@ -83,14 +95,13 @@ class AO {
       }
     }
     this.__type__ = "ao"
-
     if (ar?.__type__ === "ar") this.ar = ar
     else {
       let _ar = typeof ar === "object" ? ar : {}
-      this.ar = new AR(ar)
+      this.ar = new AR(_ar)
     }
-    if (in_memory) {
-    } else if (aoconnect) {
+
+    if (!in_memory && aoconnect) {
       const {
         results,
         assign,
@@ -620,6 +631,7 @@ class AO {
     while (attempts > 0) {
       if (!this.in_memory) await wait(1000)
       const { res, err: _err } = await this.dry({ pid, data: "#Inbox" })
+      if (res?.error) return { err: res.error, pid }
       if (typeof res?.Output === "object") break
       attempts -= 1
       if (attempts === 0) err = "timeout"

package/esm/aoconnect.js

@@ -16,6 +16,7 @@ import {
 } from "./utils.js"
 import ArMem from "./armem.js"
 import weavedrive from "./weavedrive.js"
+import kv from "./kv.js"
 import AoLoader from "@permaweb/ao-loader"
 import { readFileSync } from "fs"
 import { resolve } from "path"
@@ -35,6 +36,7 @@ export const connect = (mem, log = false) => {
   }
   const ar = new AR({ mem, log })
   const WeaveDrive = new weavedrive(ar).drive
+  const KV = new kv(ar).kv
 
   const transform = input => {
     const output = { Tags: [] }
@@ -149,9 +151,11 @@ export const connect = (mem, log = false) => {
     if (opt.item) opt.data = base64url.decode(item.data)
     await ar.postItems(item, su.jwk)
     const now = Date.now
+    const t = tags(opt.tags)
+    const wd = t.Extension === "KV" ? KV : WeaveDrive
     const handle = await AoLoader(wasm, {
       format,
-      WeaveDrive,
+      WeaveDrive: wd,
       spawn: item,
       module: mem.txs[mod],
       blockHeight: "100",
@@ -378,7 +382,6 @@ export const connect = (mem, log = false) => {
         target: opt.process,
       }))
     }
-    //await ar.postItems(item, su.jwk)
     mem.msgs[id] = opt
     await assign({
       message_item: item,
@@ -458,11 +461,11 @@ export const connect = (mem, log = false) => {
           auth: mu.addr,
         })
         function cloneMemory(memory) {
-          const buffer = memory.buffer.slice(0) // Clone the ArrayBuffer
+          const buffer = memory.buffer.slice(0)
           return new WebAssembly.Memory({
-            initial: memory.buffer.byteLength / 65536, // Memory size in pages (64KB per page)
+            initial: memory.buffer.byteLength / 65536,
             maximum: memory.maximum || undefined,
-            shared: memory.shared || false, // Retain sharing if applicable
+            shared: memory.shared || false,
           })
         }
         const res = await p.handle(p.memory, msg, _env)

package/esm/helpers.js

@@ -3,6 +3,7 @@ import assert from "assert"
 import { createDataItemSigner, connect } from "@permaweb/aoconnect"
 import { dirname as _dirname, resolve } from "path"
 import { mkdirSync, existsSync, writeFileSync, readFileSync } from "fs"
+import { optAO } from "./utils.js"
 import yargs from "yargs"
 
 let {
@@ -45,12 +46,7 @@ export class Testnet {
   constructor({ port = 4000, arweave, aoconnect, docker = false } = {}) {
     this.docker = docker
     this.arweave = arweave ?? { port }
-    this.aoconnect = aoconnect ?? {
-      MU_URL: `http://localhost:${port + 2}`,
-      SU_URL: `http://localhost:${port + 3}`,
-      CU_URL: `http://localhost:${port + 4}`,
-      GATEWAY_URL: `http://localhost:${port}`,
-    }
+    this.aoconnect = aoconnect ?? optAO(port)
     this.ar = new AR(this.arweave)
   }
   async init(jwk) {
@@ -60,7 +56,7 @@ export class Testnet {
     this.src = await new Src({ ar: this.ar }).init()
     await this.ar.init(jwk)
     this.jwk = this.ar.jwk
-    this.addr = this.ar.a
+    this.addr = this.ar.addr
     this.gql = this.ar.gql
     this.module_src = await this.src.upload("aos2_0_1", "wasm")
     this.ao = await new AO({
@@ -119,11 +115,7 @@ export const setup = async ({
 
   // ar
   arweave ??= { port: 4000 }
-  aoconnect ??= {
-    MU_URL: "http://localhost:4002",
-    CU_URL: "http://localhost:4004",
-    GATEWAY_URL: "http://localhost:4000",
-  }
+  aoconnect ??= optAO(4000)
   const ar = new AR(arweave)
   await ar.gen("10")
   const src = new Src({ ar, dir })

package/esm/kv.js

@@ -0,0 +1,226 @@
+import Arweave from "arweave"
+import { toGraphObj } from "./utils.js"
+import { map } from "ramda"
+
+const KB = 1024
+const MB = KB * 1024
+const CACHE_SZ = 32 * KB
+const CHUNK_SZ = 128 * MB
+const NOTIFY_SZ = 512 * MB
+const log = console.log
+
+export default class KV {
+  constructor(ar) {
+    this.kv = function WeaveDrive(mod, FS) {
+      return {
+        async create(id) {
+          let properties = { isDevice: false, contents: null }
+          if (!FS.analyzePath("/data/").exists) FS.mkdir("/data/")
+          let node = FS.createFile("/", "data/" + id, properties, true, false)
+          // Set initial parameters
+          const bytesLength = (await ar.data(id))?.length ?? 0
+          node.total_size = Number(bytesLength)
+          node.cache = new Uint8Array(0)
+          node.position = 0
+
+          // Add a function that defers querying the file size until it is asked the first time.
+          Object.defineProperties(node, {
+            usedBytes: {
+              get: () => bytesLength,
+            },
+          })
+
+          // Now we have created the file in the emscripten FS, we can open it as a stream
+          let stream = FS.open("/data/" + id, "r")
+
+          //console.log("JS: Created file: ", id, " fd: ", stream.fd);
+          return stream
+        },
+        async open(filename) {
+          const pathCategory = filename.split("/")[1]
+          const id = filename.split("/")[2]
+          //log("JS: Opening ID: ", id)
+          if (pathCategory === "data") {
+            if (FS.analyzePath(filename).exists) {
+              let stream = FS.open(filename, "r")
+              if (stream.fd) return stream.fd
+              console.log("JS: File not found: ", filename)
+              return 0
+            } else {
+              //console.log("JS: Open => Creating file: ", id);
+              const stream = await this.create(id)
+              //console.log("JS: Open => Created file: ", id, " fd: ", stream.fd);
+              return stream.fd
+            }
+          } else {
+            console.log("JS: Invalid path category: ", pathCategory)
+            return 0
+          }
+        },
+        async read(fd, raw_dst_ptr, raw_length) {
+          let to_read = Number(raw_length)
+          let dst_ptr = Number(raw_dst_ptr)
+          let stream = 0
+          for (let i = 0; i < FS.streams.length; i++) {
+            if (FS.streams[i].fd === fd) stream = FS.streams[i]
+          }
+
+          // Satisfy what we can with the cache first
+          let bytes_read = this.readFromCache(stream, dst_ptr, to_read)
+          stream.position += bytes_read
+          stream.lastReadPosition = stream.position
+          dst_ptr += bytes_read
+          to_read -= bytes_read
+
+          // Return if we have satisfied the request
+          if (to_read === 0) {
+            //console.log("WeaveDrive: Satisfied request with cache. Returning...")
+            return bytes_read
+          }
+          //console.log("WeaveDrive: Read from cache: ", bytes_read, " Remaining to read: ", to_read)
+
+          const chunk_download_sz = Math.max(to_read, CACHE_SZ)
+          const to = Math.min(
+            stream.node.total_size,
+            stream.position + chunk_download_sz,
+          )
+          const data = await ar.data(stream.node.name)
+          // Extract the Range header to determine the start and end of the requested chunk
+          const start = 0
+          const end = data.length
+
+          // Create a ReadableStream for the requested chunk
+          const chunk = data.subarray(start, end)
+          const response = new Response(
+            new ReadableStream({
+              start(controller) {
+                controller.enqueue(chunk) // Push the chunk to the stream
+                controller.close() // Close the stream when done
+              },
+            }),
+            {
+              headers: { "Content-Length": chunk.length.toString() },
+            },
+          )
+          const reader = response.body.getReader()
+          let bytes_until_cache = CHUNK_SZ
+          let bytes_until_notify = NOTIFY_SZ
+          let downloaded_bytes = 0
+          let cache_chunks = []
+
+          try {
+            while (true) {
+              const { done, value: chunk_bytes } = await reader.read()
+              if (done) break
+              // Update the number of downloaded bytes to be _all_, not just the write length
+              downloaded_bytes += chunk_bytes.length
+              bytes_until_cache -= chunk_bytes.length
+              bytes_until_notify -= chunk_bytes.length
+
+              // Write bytes from the chunk and update the pointer if necessary
+              const write_length = Math.min(chunk_bytes.length, to_read)
+              if (write_length > 0) {
+                //console.log("WeaveDrive: Writing: ", write_length, " bytes to: ", dst_ptr)
+                mod.HEAP8.set(chunk_bytes.subarray(0, write_length), dst_ptr)
+                dst_ptr += write_length
+                bytes_read += write_length
+                stream.position += write_length
+                to_read -= write_length
+              }
+
+              if (to_read == 0) {
+                // Add excess bytes to our cache
+                const chunk_to_cache = chunk_bytes.subarray(write_length)
+                //console.log("WeaveDrive: Cacheing excess: ", chunk_to_cache.length)
+                cache_chunks.push(chunk_to_cache)
+              }
+
+              if (bytes_until_cache <= 0) {
+                console.log(
+                  "WeaveDrive: Chunk size reached. Compressing cache...",
+                )
+                stream.node.cache = this.addChunksToCache(
+                  stream.node.cache,
+                  cache_chunks,
+                )
+                cache_chunks = []
+                bytes_until_cache = CHUNK_SZ
+              }
+
+              if (bytes_until_notify <= 0) {
+                console.log(
+                  "WeaveDrive: Downloaded: ",
+                  (downloaded_bytes / stream.node.total_size) * 100,
+                  "%",
+                )
+                bytes_until_notify = NOTIFY_SZ
+              }
+            }
+          } catch (error) {
+            console.error("WeaveDrive: Error reading the stream: ", error)
+          } finally {
+            reader.releaseLock()
+          }
+          // If we have no cache, or we have not satisfied the full request, we need to download the rest
+          // Rebuild the cache from the new cache chunks
+          stream.node.cache = this.addChunksToCache(
+            stream.node.cache,
+            cache_chunks,
+          )
+
+          // Update the last read position
+          stream.lastReadPosition = stream.position
+          return bytes_read
+        },
+        close(fd) {
+          let stream = 0
+          for (let i = 0; i < FS.streams.length; i++) {
+            if (FS.streams[i].fd === fd) stream = FS.streams[i]
+          }
+          FS.close(stream)
+        },
+
+        readFromCache(stream, dst_ptr, length) {
+          // Check if the cache has been invalidated by a seek
+          if (stream.lastReadPosition !== stream.position) {
+            //console.log("WeaveDrive: Invalidating cache for fd: ", stream.fd, " Current pos: ", stream.position, " Last read pos: ", stream.lastReadPosition)
+            stream.node.cache = new Uint8Array(0)
+            return 0
+          }
+          // Calculate the bytes of the request that can be satisfied with the cache
+          let cache_part_length = Math.min(length, stream.node.cache.length)
+          let cache_part = stream.node.cache.subarray(0, cache_part_length)
+          mod.HEAP8.set(cache_part, dst_ptr)
+          // Set the new cache to the remainder of the unused cache and update pointers
+          stream.node.cache = stream.node.cache.subarray(cache_part_length)
+
+          return cache_part_length
+        },
+
+        addChunksToCache(old_cache, chunks) {
+          // Make a new cache array of the old cache length + the sum of the chunk lengths, capped by the max cache size
+          let new_cache_length = Math.min(
+            old_cache.length +
+              chunks.reduce((acc, chunk) => acc + chunk.length, 0),
+            CACHE_SZ,
+          )
+          let new_cache = new Uint8Array(new_cache_length)
+          // Copy the old cache to the new cache
+          new_cache.set(old_cache, 0)
+          // Load the cache chunks into the new cache
+          let current_offset = old_cache.length
+          for (let chunk of chunks) {
+            if (current_offset < new_cache_length) {
+              new_cache.set(
+                chunk.subarray(0, new_cache_length - current_offset),
+                current_offset,
+              )
+              current_offset += chunk.length
+            }
+          }
+          return new_cache
+        },
+      }
+    }
+  }
+}

package/esm/utils.js

@@ -519,7 +519,21 @@ const toGraphObj = ({ query, variables }) => {
   }
   return { tar, args }
 }
+const optAO = port => {
+  return {
+    MU_URL: `http://localhost:${port + 2}`,
+    SU_URL: `http://localhost:${port + 3}`,
+    CU_URL: `http://localhost:${port + 4}`,
+    GATEWAY_URL: `http://localhost:${port}`,
+  }
+}
+const optServer = port => {
+  return { ar: port, mu: port + 2, su: port + 3, cu: port + 4 }
+}
+
 export {
+  optAO,
+  optServer,
   toGraphObj,
   jsonToStr,
   mergeChecks,