turtb 0.5.0

package/lib/turtle/connections/TurtleBranchMultiplexer.js

@@ -0,0 +1,102 @@
+import { logError } from '../../utils/logger.js'
+import { OPAQUE_UINT8ARRAY } from '../codecs/codec.js'
+import { TurtleDictionary } from '../TurtleDictionary.js'
+import { TurtleBranchUpdater } from './TurtleBranchUpdater.js'
+import { TurtleDB } from './TurtleDB.js'
+import { TurtleTalker } from './TurtleTalker.js'
+
+/**
+ * @typedef {import('../U8aTurtle.js').U8aTurtle} U8aTurtle
+ */
+
+export class TurtleBranchMultiplexer extends TurtleTalker {
+  /** @type {Object.<string, TurtleBranchUpdater>} */
+  #updatersByCpk = {}
+  #stopped = false
+
+  /**
+   * @param {string} name
+   * @param {boolean} Xours
+   * @param {TurtleDB} [turtleDB=new TurtleDB(name)]
+   * @param {Recaller} recaller
+   */
+  constructor (name, Xours, turtleDB = new TurtleDB(name), recaller) {
+    super(name, Xours, recaller)
+    this.turtleDB = turtleDB
+    this.outgoingDictionary = new TurtleDictionary(`TurtleBranchMultiplexer"${name}".outgoingDictionary`, recaller)
+    this.appendGeneratedIncomingForever() // don't await
+  }
+
+  stop () {
+    this.#stopped = true
+  }
+
+  async appendGeneratedIncomingForever () {
+    try {
+      for await (const u8aTurtle of this.incomingBranch.u8aTurtleGenerator()) {
+        const incoming = u8aTurtle.lookup()
+        if (!incoming) continue
+        const { address, /* name, */ publicKey } = incoming
+        if (!(address || publicKey)) {
+          throw new Error('address or publicKey required')
+        }
+        const uint8Array = u8aTurtle.lookup(address)
+        const turtleBranchUpdater = await this.getTurtleBranchUpdater(this.name, publicKey)
+        turtleBranchUpdater.incomingBranch.append(uint8Array)
+        // const uint8ArrayAddresses = turtleBranchUpdater.incomingBranch.lookup('uint8ArrayAddresses')
+      }
+    } catch (error) {
+      logError(() => console.error(error))
+      throw error
+    }
+  }
+
+  /**
+   * @param {Uint8Array} uint8Array
+   * @param {string} name
+   * @param {string} publicKey
+   * @param {TurtleBranchUpdater} turtleBranchUpdater
+   */
+  #sendMuxedUpdate (uint8Array, name, publicKey, turtleBranchUpdater) {
+    const address = this.outgoingDictionary.upsert(uint8Array, [OPAQUE_UINT8ARRAY])
+    const update = { address, name, publicKey }
+    this.outgoingDictionary.upsert(update)
+    this.outgoingDictionary.squash(this.outgoingBranch.index + 1)
+    this.outgoingBranch.u8aTurtle = this.outgoingDictionary.u8aTurtle
+    // const uint8ArrayAddresses = turtleBranchUpdater.outgoingBranch.lookup('uint8ArrayAddresses')
+  }
+
+  /**
+   * @param {string} name
+   * @param {string} publicKey
+   * @returns {TurtleBranchUpdater}
+   */
+  async getTurtleBranchUpdater (name = '', publicKey = '', turtleBranch) {
+    if (!name && !publicKey) throw new Error('no name or publicKey')
+    publicKey ||= name
+    name ||= publicKey
+    if (!this.#updatersByCpk[publicKey]) {
+      this.#updatersByCpk[publicKey] = (async () => {
+        // logTrace(() => console.log({ publicKey }))
+        turtleBranch ??= await this.turtleDB.summonBoundTurtleBranch(publicKey, name)
+        const updater = new TurtleBranchUpdater(name, turtleBranch, publicKey, this.Xours)
+        ;(async () => {
+          for await (const u8aTurtle of updater.outgoingBranch.u8aTurtleGenerator()) {
+            if (this.#stopped) break
+            this.#sendMuxedUpdate(u8aTurtle.uint8Array, name, publicKey, updater)
+          }
+        })()
+        updater.start()
+        return updater
+      })()
+    }
+    return this.#updatersByCpk[publicKey]
+  }
+
+  /**
+   * @type {Array.<string>}
+   */
+  get publicKeys () {
+    return Object.keys(this.#updatersByCpk)
+  }
+}

package/lib/turtle/connections/TurtleBranchMultiplexer.test.js

@@ -0,0 +1,26 @@
+import { tics } from '../../utils/nextTick.js'
+import { globalTestRunner, urlToName } from '../../utils/TestRunner.js'
+import { Signer } from '../Signer.js'
+import { Workspace } from '../Workspace.js'
+import { TurtleBranchMultiplexer } from './TurtleBranchMultiplexer.js'
+
+globalTestRunner.describe(urlToName(import.meta.url), suite => {
+  suite.it('syncs multiple SimpleAsyncTurtleBranch', async ({ assert }) => {
+    const signer = new Signer('username', 'password')
+    const name = 'test'
+    const keys = await signer.makeKeysFor(name)
+    const aTBMux = new TurtleBranchMultiplexer('a', true)
+    const aTBUpdater = await aTBMux.getTurtleBranchUpdater('test-a', keys.publicKey)
+
+    const workspace = new Workspace(name, signer, aTBUpdater.turtleBranch.recaller, aTBUpdater.turtleBranch)
+    await workspace.commit(1, 'one')
+    await tics(1) // let the commit make it into aTBUpdater's outgoing branch (or else it will start settled at 0)
+
+    const bTBMux = new TurtleBranchMultiplexer('b')
+    const bTBUpdater = await bTBMux.getTurtleBranchUpdater('test-b', keys.publicKey)
+    aTBMux.connect(bTBMux)
+
+    await bTBUpdater.settle
+    assert.equal(bTBUpdater.turtleBranch.lookup().document.message, 'one')
+  })
+})

package/lib/turtle/connections/TurtleBranchUpdater.js

@@ -0,0 +1,47 @@
+import { logDebug } from '../../utils/logger.js'
+import { Recaller } from '../../utils/Recaller.js'
+import { findCommonAncestor } from '../U8aTurtle.js'
+import { AbstractUpdater } from './AbstractUpdater.js'
+
+/** @typedef {import('../TurtleBranch.js').TurtleBranch} TurtleBranch */
+
+const allUpdaters = new Set()
+
+export class TurtleBranchUpdater extends AbstractUpdater {
+  /**
+   * @param {string} name
+   * @param {TurtleBranch} turtleBranch
+   * @param {string} publicKey
+   * @param {boolean} [Xours=false]
+   * @param {Recaller} [recaller=new Recaller(`${name}.recaller`)]
+   */
+  constructor (name, turtleBranch, publicKey, Xours, recaller = new Recaller(name)) {
+    super(name, publicKey, Xours, recaller)
+    allUpdaters.add(this)
+    this.turtleBranch = turtleBranch
+    /** @type {U8aTurtle} */
+    let lastU8aTurtle
+    this.turtleBranch.recaller.watch(name, () => {
+      if (this.turtleBranch.u8aTurtle !== lastU8aTurtle) {
+        const incomingUint8ArrayAddresses = this.incomingBranch.lookup()?.uint8ArrayAddresses
+        if (incomingUint8ArrayAddresses?.length && !this.turtleBranch.u8aTurtle?.hasAncestor?.(lastU8aTurtle)) {
+          const commonAncestor = findCommonAncestor(lastU8aTurtle, this.turtleBranch.u8aTurtle)
+          if (commonAncestor) {
+            incomingUint8ArrayAddresses.length = Math.min(incomingUint8ArrayAddresses.length, commonAncestor.index + 1)
+          } else {
+            incomingUint8ArrayAddresses.length = 0
+          }
+        }
+        logDebug(() => console.log(`TurtleBranchUpdater(#${allUpdaters.size}): ${name}`))
+        this.update(incomingUint8ArrayAddresses)
+        lastU8aTurtle = this.turtleBranch.u8aTurtle
+      }
+    })
+  }
+
+  async getUint8ArraysLength () { return this.turtleBranch.index + 1 }
+  async setUint8ArraysLength (length) { this.turtleBranch.u8aTurtle = length ? this.turtleBranch.u8aTurtle?.getAncestorByIndex?.(length - 1) : undefined }
+  async getUint8Array (index) { return this.turtleBranch.u8aTurtle?.getAncestorByIndex?.(index)?.uint8Array }
+  async pushUint8Array (uint8Array) { return this.turtleBranch.append(uint8Array) }
+  async popUint8Array () { this.turtleBranch.u8aTurtle = this.turtleBranch.u8aTurtle.parent }
+}

package/lib/turtle/connections/TurtleDB.js

@@ -0,0 +1,165 @@
+import { logDebug, logError, logWarn } from '../../utils/logger.js'
+import { Recaller } from '../../utils/Recaller.js'
+import { TurtleBranch } from '../TurtleBranch.js'
+import { Workspace } from '../Workspace.js'
+
+/**
+ * @typedef {import('../Signer.js').Signer} Signer
+ * @typedef {(turtleBranchStatus) => Promise.<void>} Binding
+ * @typedef {{
+ *   turtleBranchPromise: Promise.<TurtleBranch>,
+ *   publicKey: string,
+ *   tags: Set.<any>,
+ *   turtleBranch: TurtleBranch,
+ *   bindingInProgress: Binding,
+ *   bindings: Set.<Binding>
+ * }} TurtleBranchStatus
+ */
+
+const STATUSES_OWN_KEYS = Symbol('TurtleDB instance changed')
+
+export class TurtleDB {
+  /** @type {Object.<string, TurtleBranchStatus>} */
+  #statuses = {}
+  /** @type {Array.<Binding>} */
+  #bindings = []
+
+  /**
+   * @param {string} name
+   * @param {Recaller} recaller
+   */
+  constructor (name, recaller = new Recaller(name)) {
+    this.name = name
+    this.recaller = recaller
+  }
+
+  /**
+   * @param {Signer} signer
+   * @param {string} name
+   */
+  async makeWorkspace (signer, name) {
+    try {
+      const { publicKey } = await signer.makeKeysFor(name)
+      logDebug(() => console.log({ publicKey }))
+      const turtleBranch = await this.summonBoundTurtleBranch(publicKey, name)
+      return new Workspace(name, signer, this.recaller, turtleBranch)
+    } catch (error) {
+      logError(() => console.error(error))
+    }
+  }
+
+  /**
+   * get a TurtleBranch and init if required
+   * @param {string} publicKey
+   * @param {string} [name=publicKey]
+   * @return {Promise.<TurtleBranch>}
+   */
+  async summonBoundTurtleBranch (publicKey, name = publicKey) {
+    if (!publicKey) throw new Error('TurtleBranch must have publicKey')
+    const status = this.getStatus(publicKey, name)
+    if (status.turtleBranch.name === publicKey) status.turtleBranch.name = name
+    return status.turtleBranchPromise
+  }
+
+  /**
+   * @param {Binding} binding
+   * @returns {boolean}
+   */
+  bind (binding) {
+    if (this.#bindings.includes(binding)) {
+      logWarn(() => console.warn('binding already added'))
+      return false
+    }
+    this.#bindings.push(binding)
+    return true
+  }
+
+  /**
+   * @param {Binding} binding
+   * @returns {boolean}
+   */
+  unbind (binding) {
+    if (!this.#bindings.includes(binding)) {
+      logWarn(() => console.warn('binding already removed'))
+      return false
+    }
+    this.#bindings = this.#bindings.filter(_binding => _binding !== binding)
+    return true
+  }
+
+  /**
+   * @param {string} publicKey
+   * @param {string} tag
+   * @returns {boolean}
+   */
+  tag (publicKey, tag) {
+    if (!this.getStatus(publicKey).tags.has(tag)) {
+      this.getStatus(publicKey).tags.add(tag)
+      this.recaller.reportKeyMutation(this, STATUSES_OWN_KEYS, 'tag', this.name)
+      return true
+    }
+    return false
+  }
+
+  /**
+   * @param {string} publicKey
+   * @param {string} tag
+   * @returns {boolean}
+   */
+  untag (publicKey, tag) {
+    if (this.getStatus(publicKey).tags.has(tag)) {
+      this.getStatus(publicKey).tags.delete(tag)
+      this.recaller.reportKeyMutation(this, STATUSES_OWN_KEYS, 'untag', this.name)
+      return true
+    }
+    return false
+  }
+
+  /**
+   * @param {string} publicKey
+   * @returns {TurtleBranchStatus}
+   */
+  getStatus (publicKey, name = publicKey) {
+    this.recaller.reportKeyAccess(this, STATUSES_OWN_KEYS, 'getStatus', this.name)
+    let status = this.#statuses[publicKey]
+    if (!status) {
+      const turtleBranch = new TurtleBranch(name, this.recaller)
+      status = {
+        publicKey,
+        tags: new Set(),
+        turtleBranch,
+        bindingInProgress: null,
+        bindings: new Set()
+      }
+      this.#statuses[publicKey] = status
+      this.recaller.reportKeyMutation(this, STATUSES_OWN_KEYS, 'getStatus', this.name)
+      status.turtleBranchPromise = (async () => {
+        try {
+          for (const binding of this.#bindings) {
+            status.bindingInProgress = binding
+            await binding(status)
+            status.bindings.add(binding)
+          }
+          return turtleBranch
+        } catch (error) {
+          logError(() => console.error(error))
+        }
+      })()
+    }
+    return this.#statuses[publicKey]
+  }
+
+  /**
+   * @param {Set.<string>} tags
+   * @returns {Array.<string>}
+   */
+  getPublicKeys (tags) {
+    this.recaller.reportKeyAccess(this, STATUSES_OWN_KEYS, 'getPublicKeys', this.name)
+    const allKeys = Object.keys(this.#statuses)
+    if (!tags) return allKeys
+    return allKeys.filter(publicKey => {
+      const status = this.#statuses[publicKey]
+      return tags.intersection(status.tags).size === tags.size
+    })
+  }
+}

package/lib/turtle/connections/TurtleDB.test.js

@@ -0,0 +1,45 @@
+import { tics } from '../../utils/nextTick.js'
+import { globalTestRunner, urlToName } from '../../utils/TestRunner.js'
+import { TurtleDB } from './TurtleDB.js'
+
+globalTestRunner.describe(urlToName(import.meta.url), suite => {
+  suite.it('correctly makes branches', async ({ assert }) => {
+    const turtleDB = new TurtleDB('test1')
+    turtleDB.bind(async status => {
+      status.turtleBranch.x ??= []
+      status.turtleBranch.x.push('a')
+    })
+    turtleDB.bind(async status => {
+      status.turtleBranch.x ??= []
+      status.turtleBranch.x.push('b')
+    })
+    const branchA = await turtleDB.summonBoundTurtleBranch('abc123', 'A')
+    assert.equal(branchA.x, ['a', 'b'])
+  })
+  suite.it('filters by tag', async ({ assert }) => {
+    const turtleDB = new TurtleDB('test2')
+    await turtleDB.summonBoundTurtleBranch('a')
+    turtleDB.getStatus('a').tags.add('x')
+    await turtleDB.summonBoundTurtleBranch('b')
+    await turtleDB.summonBoundTurtleBranch('c')
+    turtleDB.tag('c', 'x')
+    const allKeys = turtleDB.getPublicKeys()
+    assert.equal(allKeys, ['a', 'b', 'c'])
+    const xKeys = turtleDB.getPublicKeys(new Set(['x']))
+    assert.equal(xKeys, ['a', 'c'])
+  })
+  suite.it('notifies on changes', async ({ assert }) => {
+    const turtleDB = new TurtleDB('test3')
+    const outputs = []
+    turtleDB.recaller.watch('test3', () => {
+      outputs.push([turtleDB.getPublicKeys(), turtleDB.getPublicKeys(new Set(['x']))])
+    })
+    await turtleDB.summonBoundTurtleBranch('a')
+    await turtleDB.summonBoundTurtleBranch('b')
+    await tics()
+    assert.equal(outputs, [[[], []], [['a', 'b'], []]])
+    turtleDB.tag('a', 'x')
+    await tics()
+    assert.equal(outputs, [[[], []], [['a', 'b'], []], [['a', 'b'], ['a']]])
+  })
+})

package/lib/turtle/connections/TurtleTalker.js

@@ -0,0 +1,34 @@
+import { Recaller } from '../../utils/Recaller.js'
+import { TurtleBranch } from '../TurtleBranch.js'
+
+/**
+ * @typedef {{ts: Date, uint8ArrayAddresses: Array.<number>}} TurtleTalk
+ */
+
+export class TurtleTalker {
+  /**
+   * @param {string} name
+   * @param {boolean} [Xours=false]
+   * @param {Recaller} [recaller=new Recaller(`${name}.recaller`)]
+   */
+  constructor (
+    name,
+    Xours = false,
+    recaller = new Recaller(`${name}.recaller`)
+  ) {
+    this.name = name
+    this.Xours = Xours
+    this.recaller = recaller
+    this.outgoingBranch = new TurtleBranch(`${name}.outgoingBranch`, recaller)
+    this.incomingBranch = new TurtleBranch(`${name}.incomingBranch`, recaller)
+  }
+
+  /** @param {TurtleTalk|TurtleBranch} connection */
+  connect (connection) {
+    this.makeReadableStream().pipeTo(connection.makeWritableStream())
+    connection.makeReadableStream().pipeTo(this.makeWritableStream())
+  }
+
+  makeReadableStream () { return this.outgoingBranch.makeReadableStream() }
+  makeWritableStream () { return this.incomingBranch.makeWritableStream() }
+}

package/lib/turtle/connections/TurtleTalker.test.js

@@ -0,0 +1,101 @@
+import { tics } from '../../utils/nextTick.js'
+import { globalTestRunner, urlToName } from '../../utils/TestRunner.js'
+import { Signer } from '../Signer.js'
+import { TurtleBranch } from '../TurtleBranch.js'
+import { Workspace } from '../Workspace.js'
+import { TurtleBranchUpdater } from './TurtleBranchUpdater.js'
+
+const commitSettle = async () => {
+  // console.log(' -- commit settle')
+  await tics(10) // , 'a sending, b verifying and updating')
+}
+
+globalTestRunner.describe(urlToName(import.meta.url), suite => {
+  suite.it('syncs SimpleAsyncTurtleBranch', async ({ assert }) => {
+    const signer = new Signer('test-user', 'p@$$w0rd')
+    const aWorkspace = new Workspace('test', signer)
+    const keys = await signer.makeKeysFor(aWorkspace.name)
+    const aTalker = new TurtleBranchUpdater('aTalker', aWorkspace.committedBranch, keys.publicKey)
+    const b = new TurtleBranch('b')
+    const bTalker = new TurtleBranchUpdater('bTalker', b, keys.publicKey, true)
+
+    aTalker.connect(bTalker)
+    aTalker.start()
+    bTalker.start()
+    await tics(1) // , 'talkers icebreaking')
+
+    await aWorkspace.commit(42, 'everything', false)
+    await commitSettle()
+    assert.equal(b.lookup()?.document?.value, 42)
+
+    await aWorkspace.commit(Math.E, 'euler', false)
+    await commitSettle()
+    assert.equal(b.lookup()?.document?.value, Math.E)
+
+    await aWorkspace.commit(Math.PI, 'pi', false)
+    await commitSettle()
+    assert.equal(b.lookup()?.document?.value, Math.PI)
+
+    const unbrokenBranch = new TurtleBranch('unbroken', undefined, aWorkspace.committedBranch.u8aTurtle)
+
+    const brokenWorkspace = new Workspace('broken', signer, aWorkspace.committedBranch.recaller, aWorkspace.committedBranch)
+    // const brokenKeys = await signer.makeKeysFor(brokenWorkspace.name)
+    await brokenWorkspace.commit(2, 'two', false)
+    await commitSettle()
+    assert.equal(b.lookup()?.document?.value, Math.PI)
+
+    const unbrokenWorkspace = new Workspace('test', signer, unbrokenBranch.recaller, unbrokenBranch)
+    await unbrokenWorkspace.commit(4, 'four', false)
+    aTalker.turtleBranch.u8aTurtle = unbrokenWorkspace.u8aTurtle
+    await commitSettle()
+    assert.equal(b.lookup()?.document?.value, 4)
+
+    const a2 = new TurtleBranch('a2', undefined, aWorkspace.committedBranch.u8aTurtle)
+    const a2Workspace = new Workspace('test', signer, a2.recaller, a2)
+    const aTalker2 = new TurtleBranchUpdater('aTalker2', a2, keys.publicKey)
+    const b2 = new TurtleBranch('b2', undefined, b.u8aTurtle)
+    const bTalker2 = new TurtleBranchUpdater('bTalker2', b2, keys.publicKey)
+    aTalker2.connect(bTalker2)
+    aTalker2.start()
+    bTalker2.start()
+    await tics(1) //, 'talkers icebreaking')
+    assert.equal(b2.lookup()?.document?.value, 4)
+    assert.isAbove(b2.length, bTalker2.outgoingBranch.length) // we have more data than we received
+
+    a2Workspace.commit(5, 'five', false)
+    await commitSettle()
+    assert.equal(b2.lookup()?.document?.value, 5)
+    assert.isAbove(b2.length, bTalker2.outgoingBranch.length) // we have more data than we received from this connection
+  })
+
+  suite.it('handles bigger changes', async ({ assert }) => {
+    const signer = new Signer('username', 'password')
+    const aWorkspace = new Workspace('test', signer)
+    const keys = await signer.makeKeysFor(aWorkspace.name)
+    await aWorkspace.commit(1, 'one', false)
+    await aWorkspace.commit(2, 'two', false)
+    await aWorkspace.commit(3, 'three', false)
+    const clone = new TurtleBranch('clone', undefined, aWorkspace.u8aTurtle.clone())
+
+    const originalTalker = new TurtleBranchUpdater('originalTalker', aWorkspace.committedBranch, keys.publicKey, true)
+    const cloneTalker = new TurtleBranchUpdater('cloneTalker', clone, keys.publicKey)
+
+    cloneTalker.connect(originalTalker)
+    await commitSettle()
+    originalTalker.start()
+    cloneTalker.start()
+    await commitSettle()
+    assert.equal(JSON.stringify(originalTalker.turtleBranch.lookup()), JSON.stringify(cloneTalker.turtleBranch.lookup()))
+    await aWorkspace.commit(4, 'four', false)
+    await commitSettle()
+    assert.equal(JSON.stringify(originalTalker.turtleBranch.lookup()), JSON.stringify(cloneTalker.turtleBranch.lookup()))
+
+    const bWorkspace = new Workspace('test', signer)
+    await bWorkspace.commit(5, 'five', false)
+    await bWorkspace.commit(6, 'six', false)
+    await bWorkspace.commit(7, 'seven', false)
+    originalTalker.turtleBranch.u8aTurtle = bWorkspace.u8aTurtle
+    await commitSettle()
+    assert.equal(JSON.stringify(originalTalker.turtleBranch.lookup()), JSON.stringify(cloneTalker.turtleBranch.lookup()))
+  })
+})