turtb 0.5.0

package/src/originSync.js

@@ -0,0 +1,75 @@
+import { createConnection } from 'net'
+import { TurtleBranchMultiplexer } from '../lib/turtle/connections/TurtleBranchMultiplexer.js'
+import { logError, logInfo, logWarn, logDebug } from '../lib/utils/logger.js'
+
+export async function originSync (turtleDB, host, port) {
+  let t = 100
+  let connectionCount = 0
+  while (true) {
+    logInfo(() => console.log('-- creating new origin connection'))
+    logInfo(() => console.time('-- origin connection lifespan'))
+    const tbMux = new TurtleBranchMultiplexer(`origin_#${host}`, false, turtleDB)
+    for (const publicKey of turtleDB.getPublicKeys()) {
+      await tbMux.getTurtleBranchUpdater(tbMux.name, publicKey)
+    }
+    const tbMuxBinding = async (/** @type {TurtleBranchStatus} */ status) => {
+      try {
+        // logDebug(() => console.log('tbMuxBinding about to get next', status.publicKey))
+        const updater = await tbMux.getTurtleBranchUpdater(tbMux.name, status.publicKey, status.turtleBranch)
+        logDebug(() => console.log('updater about to await settle', updater.name, updater.turtleBranch.length))
+        await updater.settle
+        logDebug(() => console.log('updater settled', updater.turtleBranch.length))
+      } catch (error) {
+        logError(() => console.error(error))
+      }
+    }
+    turtleDB.bind(tbMuxBinding)
+    let _connectionCount
+    try {
+      const socket = createConnection(port, host)
+      socket.on('connect', () => {
+        ;(async () => {
+          try {
+            for await (const chunk of tbMux.makeReadableStream()) {
+              if (socket.closed) break
+              if (socket.write(chunk)) {
+                // logDebug(() => console.log('host outgoing data', chunk))
+              } else {
+                logWarn(() => console.warn('socket failed to write', chunk.length, 'bytes, ending connection', chunk))
+                // break
+              }
+            }
+          } catch (error) {
+            logError(() => console.error(error))
+          }
+        })()
+        t = 100
+        _connectionCount = ++connectionCount
+        logDebug(() => console.log('-- onopen', { _connectionCount }))
+      })
+      const streamWriter = tbMux.makeWritableStream().getWriter()
+      socket.on('data', buffer => {
+        // logDebug(() => console.log('host incoming data', buffer))
+        streamWriter.write(buffer)
+      })
+      await new Promise((resolve, reject) => {
+        socket.on('close', resolve)
+        socket.on('error', reject)
+      })
+    } catch (error) {
+      if (error?.code === 'ECONNREFUSED') {
+        logWarn(() => console.log('-- connection refused'))
+      } else {
+        logError(() => console.error(error))
+        throw error
+      }
+    }
+    tbMux.stop()
+    turtleDB.unbind(tbMuxBinding)
+    logInfo(() => console.timeEnd('-- origin connection lifespan'))
+    t = Math.min(t, 2 * 60 * 1000) // 2 minutes max (unjittered)
+    t = t * (1 + Math.random()) // exponential backoff and some jitter
+    logInfo(() => console.log(`-- waiting ${(t / 1000).toFixed(2)}s`))
+    await new Promise(resolve => setTimeout(resolve, t))
+  }
+}

package/src/outletSync.js

@@ -0,0 +1,50 @@
+import { createServer } from 'net'
+import { TurtleBranchMultiplexer } from '../lib/turtle/connections/TurtleBranchMultiplexer.js'
+import { logError, logWarn, logInfo } from '../lib/utils/logger.js'
+
+export async function outletSync (turtleDB, port) {
+  let connectionCount = 0
+  const server = createServer(async socket => {
+    let tbMux
+    const _connectionCount = ++connectionCount
+    try {
+      logInfo(() => console.log('turtle connection', _connectionCount))
+      tbMux = new TurtleBranchMultiplexer(`outlet_#${_connectionCount}`, true, turtleDB)
+      ;(async () => {
+        try {
+          for await (const chunk of tbMux.makeReadableStream()) {
+            if (socket.closed) break
+            if (socket.write(chunk)) {
+              // logDebug(() => console.log('origin.host outgoing data', chunk))
+            } else {
+              await new Promise(resolve => socket.once('drain', resolve))
+            }
+          }
+        } catch (error) {
+          logError(() => console.error(error))
+        }
+      })()
+      const streamWriter = tbMux.makeWritableStream().getWriter()
+      socket.on('data', buffer => {
+        // logDebug(() => console.log('turtleHost incoming data', buffer))
+        streamWriter.write(buffer)
+      })
+
+      await new Promise((resolve, reject) => {
+        socket.on('close', resolve)
+        socket.on('error', reject)
+      })
+    } catch (error) {
+      if (error.code === 'ECONNRESET') {
+        logWarn(() => console.warn('ECONNRESET', _connectionCount))
+      } else {
+        logError(() => console.error(error))
+        throw error
+      }
+    }
+    tbMux?.stop?.()
+  })
+  server.listen(port, () => {
+    logInfo(() => console.log('opened outlet.port:', port))
+  })
+}

package/src/proxyFolder.js

@@ -0,0 +1,195 @@
+import { mkdirSync, readdirSync, readFileSync, symlinkSync, unlinkSync, writeFileSync, readlinkSync, lstatSync, rmdirSync } from 'fs'
+import { dirname, join, relative } from 'path'
+import { logError } from '../lib/utils/logger.js'
+import { deepEqual } from '../lib/utils/deepEqual.js'
+import { Recaller } from '../lib/utils/Recaller.js'
+import { subscribe } from '@parcel/watcher'
+
+export const isLinesOfTextExtension = path => path.match(/\.(html|css|js|svg|txt|gitignore|env|node_repl_history)$/)
+export const isJSONExtension = path => path.match(/\.(json)$/)
+
+export const encodeTextFile = object => {
+  if (!object || typeof object !== 'object') throw new Error('encodeFile requires an object')
+  if (Array.isArray(object) && object.every(value => typeof value === 'string')) return object.join('\n')
+  return JSON.stringify(object, undefined, 2)
+}
+
+const decodeBufferAsFileObject = (buffer, path) => {
+  if (!buffer?.isBuffer?.() && !(buffer instanceof Uint8Array)) return buffer
+  const uint8Array = new Uint8Array(buffer)
+  const isLinesOfText = isLinesOfTextExtension(path)
+  const isJSON = isJSONExtension(path)
+  if (!isLinesOfText && !isJSON) return uint8Array
+  const decoder = new TextDecoder('utf-8')
+  const str = decoder.decode(uint8Array)
+  if (isLinesOfText) return str.split(/\r?\n/)
+  try {
+    return JSON.parse(str)
+  } catch (err) {
+    logError(() => console.error(`error parsing (file:${path})`))
+    return str
+  }
+}
+
+export const equalFileObjects = (a, b, path) => {
+  const decodedA = decodeBufferAsFileObject(a, path)
+  const decodedB = decodeBufferAsFileObject(b, path)
+  return deepEqual(decodedA, decodedB)
+}
+
+export const UPDATES_HANDLER = Symbol('UPDATES_HANDLER')
+
+/**
+ * @param {string} folder
+ * @param {Recaller} recaller
+ * @param {function(string, any, any):void)}
+ * @returns {Proxy}
+ */
+export const proxyFolder = (folder, recaller = new Recaller(folder), updatesHandler) => {
+  const cleanEmptyDir = path => {
+    if (['', '.', '/'].includes(path)) return
+    const childPath = join(folder, path)
+    if (readdirSync(childPath).length) return
+    rmdirSync(childPath)
+    const parentDirname = dirname(path)
+    cleanEmptyDir(parentDirname)
+  }
+
+  const target = {}
+
+  const writeFileObject = (path, newFileObject) => {
+    const oldFileObject = target[path]
+    if (equalFileObjects(oldFileObject, newFileObject, path)) return true
+    const childPath = join(folder, path)
+    if (newFileObject) {
+      const folderPath = dirname(childPath)
+      if (folderPath) mkdirSync(folderPath, { recursive: true })
+    }
+    if (oldFileObject) unlinkSync(childPath)
+    if (!newFileObject) {
+      // no such thing, remove it
+      delete target[path]
+      cleanEmptyDir(dirname(path))
+    } else {
+      target[path] = newFileObject
+      if (newFileObject.symlink) {
+        symlinkSync(newFileObject.symlink, childPath)
+      } else if (newFileObject instanceof Uint8Array) {
+        writeFileSync(childPath, Buffer.from(newFileObject))
+      } else if (typeof newFileObject === 'object') {
+        writeFileSync(childPath, encodeTextFile(newFileObject), { encoding: 'utf8' })
+      } else {
+        throw new Error('encodeFile requires an object')
+      }
+    }
+    return true
+  }
+
+  const readFileObject = path => {
+    const childPath = join(folder, path)
+    let stats
+    try {
+      stats = lstatSync(childPath)
+    } catch (err) {
+      if (err.code !== 'ENOENT') throw err
+    }
+    let changed = ''
+    if (stats && (stats.isFile() || stats.isSymbolicLink())) {
+      let value
+      if (stats.isSymbolicLink()) {
+        const symlink = readlinkSync(childPath)
+        value = { symlink }
+      } else {
+        value = decodeBufferAsFileObject(readFileSync(childPath), childPath)
+      }
+      if (!equalFileObjects(value, target[path], path)) {
+        target[path] = value
+        changed = 'set'
+      }
+    } else {
+      if (target[path]) {
+        delete target[path]
+        changed = 'delete'
+      }
+    }
+    if (changed) {
+      recaller.reportKeyMutation(target, path, changed, `proxyFolder(${folder})`)
+      for (let parentDirname = dirname(path); !['', '.', '/'].includes(parentDirname); parentDirname = dirname(parentDirname)) {
+        recaller.reportKeyMutation(target, parentDirname, changed, `proxyFolder(${folder})`)
+      }
+    }
+    return target[path]
+  }
+
+  const readFileObjects = folder => {
+    // don't use recursive option because it follows symlinks
+    readdirSync(folder, { withFileTypes: true }).forEach(dirent => {
+      if (dirent.name.startsWith('.git/') || dirent.name.startsWith('node_modules/')) return
+      if (!dirent.isDirectory()) {
+        readFileObject(join(dirent.parentPath, dirent.name))
+      } else {
+        readFileObjects(join(folder, dirent.name))
+      }
+    })
+  }
+
+  readFileObjects(folder)
+
+  const proxy = new Proxy(target, {
+    get: (target, name) => {
+      recaller.reportKeyAccess(target, name, 'get', `proxyFolder(${folder})`)
+      if (name === UPDATES_HANDLER) {
+        return updatesHandler
+      } else if (target[name]) {
+        return target[name]
+      } else {
+        const matchingEntries = Object.entries(target).filter(([key]) => (key.startsWith(name + '/') || key === name))
+        if (!matchingEntries.length) return
+        return Object.fromEntries(matchingEntries)
+      }
+    },
+    set: (target, name, value) => {
+      if (name === UPDATES_HANDLER) {
+        updatesHandler = value
+        return true
+      }
+      return writeFileObject(name, value)
+    },
+    deleteProperty: (target, name) => {
+      recaller.reportKeyMutation(target, name, 'delete', `proxyFolder(${folder})`)
+      for (let parentDirname = dirname(name); !['', '.', '/'].includes(parentDirname); parentDirname = dirname(parentDirname)) {
+        recaller.reportKeyMutation(target, parentDirname, 'delete', `proxyFolder(${folder})`)
+      }
+      return writeFileObject(name)
+    }
+  })
+
+  let timeout
+  const modifiedFiles = new Set()
+  const handleFileChange = filename => {
+    clearTimeout(timeout)
+    modifiedFiles.add(filename)
+    timeout = setTimeout(() => {
+      const changes = {}
+      modifiedFiles.forEach(filename => {
+        const oldValue = target[filename]
+        const newValue = readFileObject(filename)
+        if (!equalFileObjects(oldValue, newValue, filename)) {
+          changes[filename] = { oldValue, newValue }
+        }
+      })
+      updatesHandler?.(changes)
+      modifiedFiles.clear()
+    }, 500)
+  }
+
+  subscribe(folder, (err, events) => {
+    if (err) throw err
+    events.forEach(({ path }) => {
+      const filename = relative(folder, path)
+      handleFileChange(filename)
+    })
+  })
+
+  return proxy
+}

package/src/s3Sync.js

@@ -0,0 +1,32 @@
+import { S3Client } from '@aws-sdk/client-s3'
+import { S3Updater } from './S3Updater.js'
+import { TurtleBranchUpdater } from '../lib/turtle/connections/TurtleBranchUpdater.js'
+import { logDebug } from '../lib/utils/logger.js'
+
+export async function s3Sync (turtleDB, recaller, endpoint, region, accessKeyId, secretAccessKey, bucket) {
+  /** @type {import('@aws-sdk/client-s3').S3ClientConfig} */
+  const s3Client = new S3Client({
+    requestHandler: { httpsAgent: { maxSockets: 500 } },
+    endpoint,
+    forcePathStyle: false,
+    region,
+    credentials: {
+      accessKeyId,
+      secretAccessKey
+    }
+  })
+  const tbMuxBinding = async (/** @type {TurtleBranchStatus} */ status) => {
+    const turtleBranch = status.turtleBranch
+    const name = turtleBranch.name
+    const publicKey = status.publicKey
+    const s3Updater = new S3Updater(`to_S3_#${name}`, publicKey, recaller, s3Client, bucket)
+    const tbUpdater = new TurtleBranchUpdater(`from_S3_#${name}`, turtleBranch, publicKey, false, recaller)
+    s3Updater.connect(tbUpdater)
+    s3Updater.start()
+    tbUpdater.start()
+    logDebug(() => console.log('tbUpdater about to await settle', tbUpdater.name))
+    if (!status.bindings.has(tbMuxBinding)) await tbUpdater.settle
+    logDebug(() => console.log('tbUpdater settled'))
+  }
+  turtleDB.bind(tbMuxBinding)
+}

package/src/webSync.js

@@ -0,0 +1,101 @@
+import express from 'express'
+import { join } from 'path'
+import { manageCert } from './manageCert.js'
+import { createServer as createHttpsServer } from 'https'
+import { createServer as createHttpServer } from 'http'
+import { WebSocketServer } from 'ws'
+import { TurtleBranchMultiplexer } from '../lib/turtle/connections/TurtleBranchMultiplexer.js'
+import { randomUUID } from 'crypto'
+import { logDebug, logInfo, logError } from '../lib/utils/logger.js'
+import { handleRedirect } from '../lib/utils/handleRedirect.js'
+
+/**
+ * @typedef {import('../lib/turtle/connections/TurtleDB.js').TurtleDB} TurtleDB
+ */
+
+const uuid = randomUUID()
+
+/**
+ * @param {number} port
+ * @param {string} basePublicKey
+ * @param {TurtleDB} turtleDB
+ * @param {boolean} https
+ * @param {boolean} insecure
+ * @param {string} certpath
+ * @param {string} [turtleDBFolder='.turtleDB']
+ */
+export async function webSync (port, basePublicKey, turtleDB, https, insecure, certpath, turtleDBFolder = '.turtleDB') {
+  const root = join(process.cwd(), basePublicKey)
+  const app = express()
+  app.use((req, _res, next) => {
+    logDebug(() => console.log(req.method, req.url, req.originalUrl))
+    next()
+  })
+  app.use(async (req, res, next) => {
+    if (req.url === '/.well-known/appspecific/com.chrome.devtools.json') {
+      res.type('application/json')
+      res.send(JSON.stringify({ workspace: { uuid, root } }))
+      return
+    }
+    handleRedirect(req.url, +req.params.address, turtleDB, basePublicKey, turtleDBFolder, href => {
+      res.redirect(302, href)
+    }, (type, body) => {
+      if (!body) return next()
+      res.type(type)
+      res.send(body)
+    })
+  })
+  // app.use(express.static(root))
+
+  let server
+  if (https) {
+    if (insecure) process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'
+    const fullcertpath = join(process.cwd(), certpath)
+    const certOptions = await manageCert(fullcertpath)
+    server = createHttpsServer(certOptions, app)
+  } else {
+    server = createHttpServer(app)
+  }
+
+  const wss = new WebSocketServer({ server })
+  let connectionCount = 0
+
+  wss.on('connection', async ws => {
+    ++connectionCount
+    const _connectionCount = connectionCount
+    logDebug(() => console.log('new connection', _connectionCount))
+    // keep alive
+    const intervalId = setInterval(() => {
+      if (_connectionCount !== connectionCount) clearInterval(intervalId)
+      else ws.send(new Uint8Array())
+    }, 20000)
+    const tbMux = new TurtleBranchMultiplexer(`ws_connection_#${connectionCount}`, true, turtleDB)
+    ;(async () => {
+      for await (const u8aTurtle of tbMux.outgoingBranch.u8aTurtleGenerator()) {
+        if (ws.readyState !== ws.OPEN) break
+        ws.send(u8aTurtle.uint8Array)
+      }
+    })()
+    ws.on('message', buffer => tbMux.incomingBranch.append(new Uint8Array(buffer)))
+    ws.on('close', (code, reason) => logDebug(() => console.log('connection closed', _connectionCount)))
+    ws.on('error', error => logError(() => console.error('connection error', { name: error.name, message: error.message })))
+    await new Promise((resolve, reject) => {
+      ws.onclose = resolve
+      ws.onerror = reject
+    })
+    clearInterval(intervalId)
+    tbMux.stop()
+  })
+
+  server.listen(port, () => {
+    logInfo(() => console.log(`local webserver started: ${https ? 'https' : 'http'}://localhost:${port}
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!! FUN-FACT: Self-signed certificates break service-workers !!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+    (HINT): On MacOS and in a browser started with this command ──╮
+               a service-worker can use a self-signed cert.       │
+╭─────────────────────────────────────────────────────────────────╯
+╰─▶ open '/Applications/Google Chrome Canary.app' --args --ignore-certificate-errors https://localhost:${port}/`))
+  })
+}