turtb 0.5.0

package/bin/turtb.js

@@ -0,0 +1,259 @@
+#!/usr/bin/env node
+
+import { readFileSync } from 'fs'
+import { start } from 'repl'
+import { Option, program } from 'commander'
+import { question, questionNewPassword } from 'readline-sync'
+import { LOG_LEVELS, logError, logInfo, logSilly, setLogLevel } from '../lib/utils/logger.js'
+import { Signer } from '../lib/turtle/Signer.js'
+import { TurtleDB } from '../lib/turtle/connections/TurtleDB.js'
+import { Recaller } from '../lib/utils/Recaller.js'
+import { OURS, THEIRS, THROW, TurtleDictionary } from '../lib/turtle/TurtleDictionary.js'
+import { Workspace } from '../lib/turtle/Workspace.js'
+import { AS_REFS } from '../lib/turtle/codecs/CodecType.js'
+import { archiveSync } from '../src/archiveSync.js'
+import { fileSync } from '../src/fileSync.js'
+import { s3Sync } from '../src/s3Sync.js'
+import { originSync } from '../src/originSync.js'
+import { outletSync } from '../src/outletSync.js'
+import { webSync } from '../src/webSync.js'
+import { config } from 'dotenv'
+
+console.log('\n\n\n!!!!!')
+const { version } = JSON.parse(readFileSync(new URL('../package.json', import.meta.url)))
+
+const defaultWebPort = 8080
+const defaultRemoteHost = 'turtledb.com'
+const defaultRemotePort = 1024
+const defaultSyncPort = 1024
+
+const makeParserWithOptions = (...options) => value => {
+  if (options.length) {
+    if (value === true) return options[0]
+    if (value === '') return options[0]
+    if (value === 'true') return options[0]
+  }
+  if (value === 'false') return false
+  if (!isNaN(+value)) value = +value
+  if (options.length <= 1 || options.includes(value)) return value
+  throw new Error(`value must be one of: ${options.join(', ')}`)
+}
+
+program
+  .name('turtledb-com')
+  .version(version)
+
+  .option('--env-file <path>', 'path to .env file')
+
+  .addOption(
+    new Option('--username <string>', 'username to use for Signer')
+      .env('TURTLEDB_USERNAME')
+  )
+  .addOption(
+    new Option('--password <string>', 'password to use for Signer')
+      .env('TURTLEDB_PASSWORD')
+  )
+  .addOption(
+    new Option('--turtlename <string>', 'name for dataset')
+      .env('TURTLEDB_TURTLENAME')
+  )
+
+  .addOption(
+    new Option('-f, --fs-mirror [resolve]', 'mirror files locally and handle')
+      .default(false)
+      .preset(THROW)
+      .choices([OURS, THEIRS, THROW, ''])
+      .argParser(makeParserWithOptions(THROW, OURS, THEIRS))
+      .env('TURTLEDB_FS_MIRROR')
+  )
+  .addOption(
+    new Option('-i, --interactive', 'flag to start repl')
+      .default(false)
+      .env('TURTLEDB_INTERACTIVE')
+  )
+  .addOption(
+    new Option('-a, --archive', 'save all turtles to files by public key')
+      .default(false)
+      .env('TURTLEDB_ARCHIVE')
+  )
+  .addOption(
+    new Option('-v, --verbose [level]', 'log data flows')
+      .default(0)
+      .preset(1)
+      .choices(Object.values(LOG_LEVELS).map(v => v.toString()))
+      .argParser(makeParserWithOptions(1, ...Object.values(LOG_LEVELS)))
+      .env('TURTLEDB_VERBOSE')
+  )
+  .addOption(
+    new Option('--turtleDB-folder <path>', 'path to folder for TurtleDB files')
+      .default('.turtleDB')
+      .env('TURTLEDB_FOLDER')
+  )
+
+  .addOption(
+    new Option('-w, --web-port [number]', 'web port to sync from')
+      .default(false)
+      .preset(defaultWebPort)
+      .argParser(makeParserWithOptions(defaultWebPort))
+      .env('TURTLEDB_WEB_PORT')
+      .helpGroup('Web Server:')
+  )
+  .addOption(
+    new Option('--web-fallback <string>', 'project public key to use as fallback for web')
+      .env('TURTLEDB_WEB_FALLBACK')
+      .helpGroup('Web Server:')
+  )
+  .addOption(
+    new Option('--web-certpath <string>', 'path to self-cert for web')
+      .env('TURTLEDB_WEB_CERTPATH')
+      .helpGroup('Web Server:')
+  )
+  .addOption(
+    new Option('--web-insecure', '(local dev) allow unauthorized for web')
+      .env('TURTLEDB_WEB_INSECURE')
+      .helpGroup('Web Server:')
+  )
+
+  .addOption(
+    new Option('--remote-host [string]', 'remote host to sync to')
+      .default(false)
+      .preset(defaultRemoteHost)
+      .argParser(makeParserWithOptions(defaultRemoteHost))
+      .env('TURTLEDB_REMOTE_HOST')
+      .helpGroup('TurtleDB Syncing:')
+  )
+  .addOption(
+    new Option('-r, --remote-port [number]', 'remote port to sync to')
+      .default(false)
+      .preset(defaultRemotePort)
+      .argParser(makeParserWithOptions(defaultRemotePort))
+      .env('TURTLEDB_REMOTE_PORT')
+      .helpGroup('TurtleDB Syncing:')
+  )
+
+  .addOption(
+    new Option('-p, --sync-port [number]', 'local port to sync from')
+      .default(false)
+      .preset(defaultSyncPort)
+      .argParser(makeParserWithOptions(defaultSyncPort))
+      .env('TURTLEDB_PORT')
+      .helpGroup('TurtleDB Syncing:')
+  )
+
+  .addOption(
+    new Option('--s3-end-point <string>', 'endpoint for s3 (like "https://sfo3.digitaloceanspaces.com")')
+      .default(false)
+      .argParser(makeParserWithOptions())
+      .env('TURTLEDB_S3_END_POINT')
+      .helpGroup('S3-like Service Syncing:')
+  )
+  .addOption(
+    new Option('--s3-region <string>', 'region for s3 (like "sfo3")')
+      .env('TURTLEDB_S3_REGION')
+      .helpGroup('S3-like Service Syncing:')
+  )
+  .addOption(
+    new Option('--s3-access-key-id <string>', 'accessKeyId for s3')
+      .env('TURTLEDB_S3_ACCESS_KEY_ID')
+      .helpGroup('S3-like Service Syncing:')
+  )
+  .addOption(
+    new Option('--s3-secret-access-key <string>', 'secretAccessKey for s3')
+      .env('TURTLEDB_S3_SECRET_ACCESS_KEY')
+      .helpGroup('S3-like Service Syncing:')
+  )
+  .addOption(
+    new Option('--s3-bucket <string>', 'bucket for s3')
+      .env('TURTLEDB_S3_BUCKET')
+      .helpGroup('S3-like Service Syncing:')
+  )
+
+  .parse()
+
+const options = program.opts()
+if (options.envFile) {
+  config({ path: options.envFile })
+  program.parse() // re-parse with new env vars
+  Object.assign(options, program.opts()) // update options with new env vars
+}
+setLogLevel(options.verbose)
+let username = options.username
+let turtlename = options.turtlename
+let signer
+if (options.fsMirror !== false) {
+  username ||= question('Username: ')
+  turtlename ||= question('Turtlename: ')
+  signer = new Signer(username, options.password || questionNewPassword('Password [ATTENTION!: Backspace won\'t work here]: ', { min: 4, max: 999 }))
+} else if (username && turtlename && options.password) {
+  signer = new Signer(username, options.password)
+}
+const publicKey = (await signer.makeKeysFor(turtlename)).publicKey
+logInfo(() => console.log({ username, turtlename, publicKey }))
+
+logSilly(() => console.log({ options }))
+// console.log({ options })
+// process.exit(0)
+
+const recaller = new Recaller('turtledb-com')
+const turtleDB = new TurtleDB('turtledb-com', recaller)
+
+if (options.syncPort !== false) {
+  const syncPort = +options.syncPort || defaultSyncPort
+  logInfo(() => console.log(`listening for local connections on port ${syncPort}`))
+  outletSync(turtleDB, syncPort)
+}
+
+if (options.remoteHost !== false || options.remotePort !== false) {
+  const remoteHost = options.remoteHost || defaultRemoteHost
+  const remotePort = +options.remotePort || defaultRemotePort
+  logInfo(() => console.log(`connecting to remote at ${remoteHost}:${remotePort}`))
+  originSync(turtleDB, remoteHost, remotePort)
+}
+
+if (options.s3EndPoint !== false) {
+  s3Sync(turtleDB, recaller, options.s3EndPoint, options.s3Region, options.s3AccessKeyId, options.s3SecretAccessKey, options.s3Bucket)
+}
+
+if (options.archive) {
+  const archivePath = `${options.turtleDBFolder}/archive`
+  logInfo(() => console.log(`archiving to ${archivePath}`))
+  archiveSync(turtleDB, recaller, archivePath)
+}
+
+if (options.fsMirror !== false) {
+  if (![OURS, THEIRS, THROW].includes(options.fsMirror)) {
+    logError(() => console.error(`fs-mirror resolve option must be "${OURS}", "${THEIRS}" or "${THROW}" (you provided: "${options.fsMirror}")`))
+    process.exit(1)
+  }
+  logInfo(() => console.log('mirroring to file system'))
+  fileSync(turtlename, turtleDB, signer, undefined, options.fsMirror, options.turtleDBFolder)
+}
+
+if (options.webPort !== false) {
+  const webPort = +options.webPort
+  const insecure = !!options.webInsecure
+  const https = insecure || !!options.webCertpath
+  const certpath = options.webCertpath || `${options.turtleDBFolder}/dev/cert.json`
+  logInfo(() => console.log(`listening for web connections on port ${webPort} (https: ${https}, insecure: ${insecure}, certpath: ${certpath})`))
+  webSync(webPort, publicKey, turtleDB, https, insecure, certpath, options.turtleDBFolder)
+}
+
+if (options.interactive) {
+  global.username = username
+  global.turtlename = turtlename
+  global.signer = signer
+  global.publicKey = publicKey
+  global.recaller = recaller
+  global.turtleDB = turtleDB
+  global.workspace = await turtleDB.makeWorkspace(signer, turtlename)
+  global.TurtleDictionary = TurtleDictionary
+  global.Signer = Signer
+  global.Workspace = Workspace
+  global.setLogLevel = setLogLevel
+  global.AS_REFS = AS_REFS
+  const replServer = start({ breakEvalOnSigint: true })
+  replServer.setupHistory('.node_repl_history', err => {
+    if (err) logError(() => console.error(err))
+  })
+  replServer.on('exit', process.exit)
+}

package/lib/display/h.js

@@ -0,0 +1,204 @@
+const _voidElements = new Set(['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', 'track', 'wbr'])
+const END = Symbol('end')
+
+function _assertChar (arr, regex) {
+  if (!arr[arr.i].match(regex)) {
+    throw new Error(`expected ${regex}. got ${arr[arr.i]} at i=${arr.i}`)
+  }
+  arr.i++
+}
+
+function _readValue (arr) {
+  if (arr[arr.i].isValue) {
+    return arr[arr.i++]
+  }
+}
+
+function _readEscaped (arr) {
+  _assertChar(arr, /&/)
+  if (_readIf(arr, 'amp;')) {
+    return '&'
+  } else if (_readIf(arr, 'apos;')) {
+    return '\''
+  } else if (_readIf(arr, 'gt;')) {
+    return '>'
+  } else if (_readIf(arr, 'lt;')) {
+    return '<'
+  } else if (_readIf(arr, 'quot;')) {
+    return '"'
+  } else if (_readIf(arr, 'nbsp;')) {
+    return ' '
+  } else {
+    throw new Error('unhandled escape sequence')
+  }
+}
+
+function _readTo (arr, regex) {
+  const ss = []
+  while (arr.i < arr.length) {
+    const c = arr[arr.i]
+    if (c.isValue || c.match(regex)) {
+      return ss.join('')
+    } else if (c === '&') {
+      ss.push(_readEscaped(arr))
+    } else {
+      ss.push(c)
+      arr.i++
+    }
+  }
+  return ss.join('')
+}
+
+function _skipWhiteSpace (arr) {
+  _readTo(arr, /\S/)
+}
+
+function _readIf (arr, str) {
+  if (!str.length) {
+    str = [str]
+  }
+  const out = []
+  for (let i = 0; i < str.length; i++) {
+    const char = arr[arr.i + i]
+    if (!char || !char.match || !char.match(str[i])) {
+      return false
+    }
+    out.push(char)
+  }
+  arr.i += str.length
+  return out.join('')
+}
+
+function _readValueParts (arr, regex) {
+  const out = []
+  let ss = []
+  while (arr.i < arr.length) {
+    const c = arr[arr.i]
+    if (c.isValue) {
+      if (ss.length) {
+        out.push({ type: 'part', value: ss.join('') })
+        ss = []
+      }
+      out.push(c.value)
+      arr.i++
+    } else if (c.match(regex)) {
+      if (ss.length) {
+        out.push({ type: 'part', value: ss.join('') })
+      }
+      return out
+    } else if (c === '&') {
+      ss.push(_readEscaped(arr))
+    } else {
+      ss.push(c)
+      arr.i++
+    }
+  }
+}
+
+function _decodeAttribute (arr) {
+  _skipWhiteSpace(arr)
+  const c = arr[arr.i]
+  if (c === '/' || c === '>') {
+    return END
+  }
+  let name = _readValue(arr)
+  if (name && name.isValue) {
+    return name.value
+  }
+  name = _readTo(arr, /[\s=/>]/)
+  if (!name) {
+    throw new Error('attribute must have a name (dynamic attributes okay, dynamic names... sorry)')
+  }
+  _skipWhiteSpace(arr)
+  const equalSign = _readIf(arr, '=')
+  if (equalSign) {
+    _skipWhiteSpace(arr)
+    let value = _readValue(arr)
+    if (value) {
+      value = value.value
+    } else {
+      const quote = _readIf(arr, /['"]/)
+      if (quote) {
+        value = _readValueParts(arr, quote)
+        _assertChar(arr, quote)
+      } else {
+        value = _readTo(arr, /[\s=/>]/)
+      }
+    }
+    return { type: 'attribute', name, value }
+  } else {
+    return { type: 'attribute', name }
+  }
+}
+
+function _decodeAttributes (arr) {
+  const attributes = []
+  while (true) {
+    const attribute = _decodeAttribute(arr)
+    if (attribute !== END) {
+      attributes.push(attribute)
+    } else {
+      return attributes
+    }
+  }
+}
+
+function _decodeTag (arr) {
+  _skipWhiteSpace(arr)
+  const c = arr[arr.i]
+  if (c.isValue) {
+    arr.i++
+    return c.value
+  }
+  return _readTo(arr, /[\s/>]/)
+}
+
+function _decodeElement (arr) {
+  const c = arr[arr.i]
+  if (c.isValue) {
+    arr.i++
+    return c.value
+  } else if (c === '<') {
+    _assertChar(arr, /</)
+    const isClosing = _readIf(arr, '/')
+    const tag = _decodeTag(arr)
+    const isVoid = _voidElements.has(tag)
+    const attributes = _decodeAttributes(arr)
+    const isEmpty = _readIf(arr, '/') || isVoid
+    _assertChar(arr, />/)
+    const children = (isClosing || isEmpty) ? [] : _decodeElements(arr, tag)
+    if (isVoid && isClosing) return null
+    return { type: 'node', tag, attributes, children, isClosing }
+  } else {
+    return { type: 'textnode', value: _readTo(arr, /</) }
+  }
+}
+
+function _decodeElements (arr, closingTag) {
+  const nodes = []
+  while (arr.i < arr.length) {
+    const node = _decodeElement(arr)
+    if (node != null) {
+      if (node.isClosing) {
+        if (closingTag != null) {
+          return nodes
+        }
+      } else {
+        delete node.isClosing
+        nodes.push(node)
+      }
+    }
+  }
+  return [].concat.apply([], nodes)
+}
+
+export function h (strings, ...values) {
+  const ss = [strings[0].split('')]
+  for (let i = 0; i < values.length; i++) {
+    ss.push({ value: values[i], isValue: true })
+    ss.push(strings[i + 1].split(''))
+  }
+  const arr = [].concat.apply([], ss)
+  arr.i = 0
+  return _decodeElements(arr, null)
+}

package/lib/display/helpers.js

@@ -0,0 +1,78 @@
+// takes a function that evaluates to true or false and static if and else responses. return function that evaluates condition and returns appropriate response
+export function showIfElse (condition, a, b = []) {
+  return () => condition() ? a : b
+}
+
+// takes an object and a mapping-function. returns a function that maps the object through the function and returns previously calculated mappings on future calls
+export function mapEntries (object, f) {
+  let cache = {}
+  return () => {
+    const o = (typeof object === 'function' ? object() : object) || {}
+    const newCache = {}
+    const mappedEntries = Object.entries(o).map(([name, input]) => {
+      if (cache[name] && cache[name].input === input) {
+        newCache[name] = cache[name]
+      } else {
+        newCache[name] = {
+          input,
+          output: f(input, name)
+        }
+      }
+      return newCache[name].output
+    })
+    cache = newCache
+    return mappedEntries
+  }
+}
+
+// TODO: this only works for unique outputs... fix that
+/*
+    export function looselyMapEntries(object, f) {
+      const map = new Map()
+      return () => {
+        const o = (typeof object === 'function' ? object() : object) || {}
+        map.forEach(indexedValues => {
+          indexedValues.index = 0
+        })
+        const mappedEntries = Object.entries(o).map(([name, value]) => {
+          if (!map.has(value)) {
+            map.set(value, { index: 0, values: [] })
+          }
+          const indexedValues = map.get(value)
+          if (indexedValues.values.length <= indexedValues.index) {
+            indexedValues.values.push(f(value, name))
+          }
+          return indexedValues.values[indexedValues.index++]
+        })
+        // cleanup any unmapped-to values
+        map.forEach((indexedValues, key) => {
+          if (indexedValues.index) {
+            indexedValues.values.splice(indexedValues.index)
+          } else {
+            map.delete(key)
+          }
+        })
+        return mappedEntries
+      }
+    }
+    */
+
+export function mapSwitch (expression, f) {
+  const map = new Map()
+  return () => {
+    const expr = expression()
+    if (!map.has(expr)) {
+      map.set(expr, f(expr))
+    }
+    return map.get(expr)
+  }
+}
+
+export function objToDeclarations (obj = {}) {
+  return Object.entries(obj).map(([name, value]) => `${name}: ${value};`).join('')
+}
+
+// people just don't like writing 'el => e => {...}'
+export function handle (handler, ...args) {
+  return el => e => handler(e, el, ...args)
+}