teamplay 0.0.1

package/orm/dataTree.js

@@ -0,0 +1,196 @@
+import { observable, raw } from '@nx-js/observer-util'
+import jsonDiff from 'json0-ot-diff'
+import diffMatchPatch from 'diff-match-patch'
+import { getConnection } from './connection.js'
+
+const ALLOW_PARTIAL_DOC_CREATION = false
+
+export const dataTreeRaw = {}
+const dataTree = observable(dataTreeRaw)
+
+export function get (segments, tree = dataTree) {
+  let dataNode = tree
+  for (const segment of segments) {
+    if (dataNode == null) return dataNode
+    dataNode = dataNode[segment]
+  }
+  return dataNode
+}
+
+export function getRaw (segments) {
+  return get(segments, dataTreeRaw)
+}
+
+export function set (segments, value, tree = dataTree) {
+  let dataNode = tree
+  let dataNodeRaw = raw(dataTree)
+  for (let i = 0; i < segments.length - 1; i++) {
+    const segment = segments[i]
+    if (dataNode[segment] == null) {
+      // if next segment is a number, it means that we are in the array
+      if (typeof segments[i + 1] === 'number') dataNode[segment] = []
+      else dataNode[segment] = {}
+    }
+    dataNode = dataNode[segment]
+    dataNodeRaw = dataNodeRaw[segment]
+  }
+  const key = segments[segments.length - 1]
+  // handle adding out of bounds empty element to the array
+  if (value == null && Array.isArray(dataNodeRaw) && key >= dataNodeRaw.length) {
+    // inject new undefined elements to the end of the array
+    dataNode.splice(dataNodeRaw.length, key - dataNodeRaw.length + 1,
+      ...Array(key - dataNodeRaw.length + 1).fill(undefined))
+    return
+  }
+  // handle when the value didn't change
+  if (value === dataNodeRaw[key]) return
+  // handle setting undefined value
+  if (value == null) {
+    if (Array.isArray(dataNodeRaw)) {
+      // if parent is an array -- we set array element to undefined
+      // IMPORTANT: JSON serialization will replace `undefined` with `null`
+      //            so if the data will go to the server, it will be serialized as `null`.
+      //            And when it comes back from the server it will be still `null`.
+      //            This can lead to confusion since when you set `undefined` the value
+      //            might end up becoming `null` for seemingly no reason (like in this case).
+      dataNode[key] = undefined
+    } else {
+      // if parent is an object -- we completely delete the property.
+      // Deleting the property is better for the JSON serialization
+      // since JSON does not have `undefined` values and replaces them with `null`.
+      delete dataNode[key]
+    }
+    return
+  }
+  // just set the new value
+  dataNode[key] = value
+}
+
+export function del (segments, tree = dataTree) {
+  let dataNode = tree
+  for (let i = 0; i < segments.length - 1; i++) {
+    const segment = segments[i]
+    if (dataNode[segment] == null) return
+    dataNode = dataNode[segment]
+  }
+  if (Array.isArray(dataNode)) {
+    // remove the element from the array
+    dataNode.splice(segments[segments.length - 1], 1)
+  } else {
+    // remove the property from the object
+    delete dataNode[segments[segments.length - 1]]
+  }
+}
+
+export async function setPublicDoc (segments, value, deleteValue = false) {
+  if (segments.length === 0) throw Error(ERRORS.publicDoc(segments))
+  if (segments.length === 1) {
+    // set multiple documents at the same time
+    if (typeof value !== 'object') throw Error(ERRORS.notObjectCollection(segments, value))
+    for (const docId in value) {
+      await setPublicDoc([segments[0], docId], value[docId])
+    }
+  }
+  const [collection, docId] = segments
+  if (typeof docId === 'number') throw Error(ERRORS.publicDocIdNumber(segments))
+  if (docId === 'undefined') throw Error(ERRORS.publicDocIdUndefined(segments))
+  if (!(collection && docId)) throw Error(ERRORS.publicDoc(segments))
+  const doc = getConnection().get(collection, docId)
+  if (!doc.data && deleteValue) throw Error(ERRORS.deleteNonExistentDoc(segments))
+  // make sure that the value is not observable to not trigger extra reads. And clone it
+  value = raw(value)
+  if (value == null) {
+    value = undefined
+  } else {
+    value = JSON.parse(JSON.stringify(value))
+  }
+  if (segments.length === 2 && !doc.data) {
+    // > create a new doc. Full doc data is provided
+    if (typeof value !== 'object') throw Error(ERRORS.notObject(segments, value))
+    const newDoc = value
+    return new Promise((resolve, reject) => {
+      doc.create(newDoc, err => err ? reject(err) : resolve())
+    })
+  } else if (!doc.data) {
+    // >> create a new doc. Partial doc data is provided (subpath)
+    // NOTE: We throw an error when trying to set a subpath on a non-existing doc
+    //       to prevent potential mistakes. In future we might allow it though.
+    if (!ALLOW_PARTIAL_DOC_CREATION) throw Error(ERRORS.partialDocCreation(segments, value))
+    const newDoc = {}
+    set(segments.slice(2), value, newDoc)
+    return new Promise((resolve, reject) => {
+      doc.create(newDoc, err => err ? reject(err) : resolve())
+    })
+  } else if (segments.length === 2 && (deleteValue || value == null)) {
+    // > delete doc
+    return new Promise((resolve, reject) => {
+      doc.del(err => err ? reject(err) : resolve())
+    })
+  } else if (segments.length === 2) {
+    // > modify existing doc. Full doc modification
+    if (typeof value !== 'object') throw Error(ERRORS.notObject(segments, value))
+    const oldDoc = getRaw([collection, docId])
+    const diff = jsonDiff(oldDoc, value, diffMatchPatch)
+    return new Promise((resolve, reject) => {
+      doc.submitOp(diff, err => err ? reject(err) : resolve())
+    })
+  } else {
+    // > modify existing doc. Partial doc modification
+    const oldDoc = getRaw([collection, docId])
+    const newDoc = JSON.parse(JSON.stringify(oldDoc))
+    if (deleteValue) {
+      del(segments.slice(2), newDoc)
+    } else {
+      set(segments.slice(2), value, newDoc)
+    }
+    const diff = jsonDiff(oldDoc, newDoc, diffMatchPatch)
+    return new Promise((resolve, reject) => {
+      doc.submitOp(diff, err => err ? reject(err) : resolve())
+    })
+  }
+}
+
+export default dataTree
+
+const ERRORS = {
+  publicDoc: segments => `Public doc should have collection and docId. Got: ${segments}`,
+  nonExistingDoc: segments => `
+    Trying to modify a non-existing doc ${segments}.
+    Make sure you have subscribed to the doc before modifying it OR creating it.
+  `,
+  notObject: (segments, value) => `
+    Trying to set a non-object value to a public doc ${segments}.
+    Value: ${value}
+  `,
+  notObjectCollection: (segments, value) => `
+    Trying to set multiple documents for the collection but the value passed is not an object.
+    Path: ${segments}
+    Value: ${value}
+  `,
+  publicDocIdNumber: segments => `
+    Public doc id must be a string. Got a number: ${segments}
+  `,
+  deleteNonExistentDoc: segments => `
+    Trying to delete data from a non-existing doc ${segments}.
+    Make sure that the document exists and you are subscribed to it
+    before trying to delete anything from it or the doc itself.
+  `,
+  publicDocIdUndefined: segments => `
+    Trying to modify a public document with the id 'undefined'.
+    It's most likely a bug in your code and the variable you are using to store
+    the document id is not initialized correctly.
+    Got path: ${segments}
+  `,
+  partialDocCreation: (segments, value) => `
+    Can't set a value to a subpath of a document which doesn't exist.
+
+    You have probably forgotten to subscribe to the document.
+    You MUST subscribe to an existing document with 'sub()' before trying to modify it.
+
+    If instead you want to create a new document, you must provide the full data for it
+    and set it for the $.collection.docId signal.
+
+    Path: ${segments}
+    Value: ${value}
+  `
+}

package/orm/getSignal.js

@@ -0,0 +1,100 @@
+import Cache from './Cache.js'
+import Signal, { regularBindings, extremelyLateBindings, isPublicCollection, isPrivateCollection } from './Signal.js'
+import { findModel } from './addModel.js'
+import { LOCAL } from './$.js'
+import { ROOT, ROOT_ID, GLOBAL_ROOT_ID } from './Root.js'
+import { QUERIES } from './Query.js'
+
+const PROXIES_CACHE = new Cache()
+const PROXY_TO_SIGNAL = new WeakMap()
+
+// extremely late bindings let you use fields in your raw data which have the same name as signal's methods
+const USE_EXTREMELY_LATE_BINDINGS = true
+
+// get proxy-wrapped signal from cache or create a new one
+// TODO: move Private, Public, Local signals out of this file, same as Query has its own signal
+export default function getSignal ($root, segments = [], {
+  useExtremelyLateBindings = USE_EXTREMELY_LATE_BINDINGS,
+  rootId,
+  signalHash,
+  proxyHandlers = getDefaultProxyHandlers({ useExtremelyLateBindings })
+} = {}) {
+  if (!($root instanceof Signal)) {
+    if (segments.length === 0 && !rootId) throw Error(ERRORS.rootIdRequired)
+    if (segments.length >= 1 && isPrivateCollection(segments[0])) {
+      if (segments[0] === QUERIES) {
+        // TODO: this is a hack to temporarily let the queries work.
+        //       '$queries' collection is always added to the global (singleton) root signal.
+        //       In future it should also be part of the particular root signal.
+        $root = getSignal(undefined, [], { rootId: GLOBAL_ROOT_ID })
+      } else {
+        throw Error(ERRORS.rootSignalRequired)
+      }
+    }
+  }
+  signalHash ??= hashSegments(segments, $root?.[ROOT_ID] || rootId)
+  let proxy = PROXIES_CACHE.get(signalHash)
+  if (proxy) return proxy
+
+  const SignalClass = getSignalClass(segments)
+  const signal = new SignalClass(segments)
+  proxy = new Proxy(signal, proxyHandlers)
+  if (segments.length >= 1) {
+    if (isPrivateCollection(segments[0])) {
+      proxy[ROOT] = $root
+    } else {
+      // TODO: this is probably a hack, currently public collection signals don't need a root signal
+      //       but without it calling the methods of root signal like $.get() doesn't work
+      proxy[ROOT] = $root || getSignal(undefined, [], { rootId: GLOBAL_ROOT_ID })
+    }
+  }
+  PROXY_TO_SIGNAL.set(proxy, signal)
+  const dependencies = []
+
+  // if the signal is a child of the local value created through the $() function,
+  // we need to add the parent signal ('$local.id') to the dependencies so that it doesn't get garbage collected
+  // before the child signal ('$local.id.firstName') is garbage collected.
+  // Same goes for public collections -- we need to keep the document signal alive while its child signals are alive
+  if (segments.length > 2) {
+    if (segments[0] === LOCAL) {
+      dependencies.push(getSignal($root, segments.slice(0, 2)))
+    } else if (isPublicCollection(segments[0])) {
+      dependencies.push(getSignal(undefined, segments.slice(0, 2)))
+    }
+  }
+
+  PROXIES_CACHE.set(signalHash, proxy, dependencies)
+  return proxy
+}
+
+function getDefaultProxyHandlers ({ useExtremelyLateBindings } = {}) {
+  return useExtremelyLateBindings ? extremelyLateBindings : regularBindings
+}
+
+function hashSegments (segments, rootId) {
+  if (segments.length === 0) {
+    if (!rootId) throw Error(ERRORS.rootIdRequired)
+    return JSON.stringify({ root: rootId })
+  } else if (isPrivateCollection(segments[0])) {
+    if (!rootId) throw Error(ERRORS.privateCollectionRootIdRequired(segments))
+    return JSON.stringify({ private: [rootId, segments] })
+  } else {
+    return JSON.stringify(segments)
+  }
+}
+
+export function getSignalClass (segments) {
+  return findModel(segments) ?? Signal
+}
+
+export function rawSignal (proxy) {
+  return PROXY_TO_SIGNAL.get(proxy)
+}
+
+export { PROXIES_CACHE as __DEBUG_SIGNALS_CACHE__ }
+
+const ERRORS = {
+  rootIdRequired: 'Root signal must have a rootId specified',
+  privateCollectionRootIdRequired: segments => `Private collection signal must have a rootId specified. Segments: ${segments}`,
+  rootSignalRequired: 'First argument of getSignal() for private collections must be a Root Signal'
+}

package/orm/sub.js

@@ -0,0 +1,32 @@
+import Signal, { SEGMENTS, isPublicCollectionSignal, isPublicDocumentSignal } from './Signal.js'
+import { docSubscriptions } from './Doc.js'
+import { querySubscriptions, getQuerySignal } from './Query.js'
+
+export default function sub ($signal, params) {
+  if (isPublicDocumentSignal($signal)) {
+    return doc$($signal)
+  } else if (isPublicCollectionSignal($signal)) {
+    return query$($signal, params)
+  } else if (typeof $signal === 'function' && !($signal instanceof Signal)) {
+    return api$($signal, params)
+  } else {
+    throw Error('Invalid args passed for sub()')
+  }
+}
+
+function doc$ ($doc) {
+  const promise = docSubscriptions.subscribe($doc)
+  if (!promise) return $doc
+  return new Promise(resolve => promise.then(() => resolve($doc)))
+}
+
+function query$ ($collection, params) {
+  const $query = getQuerySignal($collection[SEGMENTS], params)
+  const promise = querySubscriptions.subscribe($query)
+  if (!promise) return $query
+  return new Promise(resolve => promise.then(() => resolve($query)))
+}
+
+function api$ (fn, args) {
+  throw Error('sub() for async functions is not implemented yet')
+}

package/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "teamplay",
+  "version": "0.0.1",
+  "description": "Deep signals ORM for React with real-time collaboration",
+  "type": "module",
+  "main": "index.js",
+  "exports": {
+    ".": "./index.js",
+    "./connect": "./connect/index.js",
+    "./server": "./server.js",
+    "./connect-test": "./connect/test.js"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "test": "npm run test-server && npm run test-client",
+    "test-server": "node --expose-gc -r ./test/_init.cjs --test",
+    "test-server-only": "node --expose-gc -r ./test/_init.cjs --test --test-only",
+    "test-client": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest"
+  },
+  "dependencies": {
+    "@nx-js/observer-util": "^4.1.3",
+    "@startupjs/cache": "^0.56.0-alpha.0",
+    "@startupjs/channel": "^0.56.0-alpha.0",
+    "@startupjs/debug": "^0.56.0-alpha.0",
+    "@startupjs/utils": "^0.56.0-alpha.64",
+    "diff-match-patch": "^1.0.5",
+    "json0-ot-diff": "^1.1.2",
+    "lodash": "^4.17.20",
+    "sharedb": "^5.0.0",
+    "uuid": "^9.0.1"
+  },
+  "devDependencies": {
+    "@jest/globals": "^29.7.0",
+    "@testing-library/react": "^15.0.7",
+    "jest": "^29.7.0",
+    "jest-environment-jsdom": "^29.7.0",
+    "sharedb-mingo-memory": "^3.0.1"
+  },
+  "peerDependencies": {
+    "react": "*",
+    "sharedb-mingo-memory": "*"
+  },
+  "peerDependenciesMeta": {
+    "sharedb-mingo-memory": {
+      "optional": true
+    }
+  },
+  "jest": {
+    "transform": {},
+    "testEnvironment": "jsdom",
+    "testRegex": "test_client/.*\\.jsx?$",
+    "testPathIgnorePatterns": [
+      "node_modules",
+      "<rootDir>/test_client/helpers"
+    ]
+  },
+  "license": "MIT"
+}

package/react/convertToObserver.js

@@ -0,0 +1,93 @@
+// TODO: rewrite to use useSyncExternalStore like in mobx. This will also help with handling Suspense abandonment better
+//       to cleanup the observer() reaction when the component is unmounted or was abandoned and unmounts will never trigger.
+//       ref: https://github.com/mobxjs/mobx/blob/94bc4997c14152ff5aefcaac64d982d5c21ba51a/packages/mobx-react-lite/src/useObserver.ts
+import { createElement as el, forwardRef as _forwardRef, useCallback, useState, useMemo } from 'react'
+import _throttle from 'lodash/throttle.js'
+import { observe, unobserve } from '@nx-js/observer-util'
+import { pipeComponentMeta, useCache, useUnmount } from './helpers.js'
+import trapRender from './trapRender.js'
+
+const DEFAULT_THROTTLE_TIMEOUT = 100
+
+export default function convertToObserver (BaseComponent, options = {}) {
+  options = { ...DEFAULT_OPTIONS, ...options }
+  const { forwardRef } = options
+  // MAGIC. This fixes hot-reloading. TODO: figure out WHY it fixes it
+  const random = Math.random()
+
+  // memo; we are not intested in deep updates
+  // in props; we assume that if deep objects are changed,
+  // this is in observables, which would have been tracked anyway
+  let Component = (...args) => {
+    // forceUpdate 2.0
+    const forceUpdate = useForceUpdate(options.throttle)
+    const cache = useCache(options.cache != null ? options.cache : true)
+
+    // wrap the BaseComponent into an observe decorator once.
+    // This way it will track any observable changes and will trigger rerender
+    const reactionRef = useMemo(() => ({}), [])
+    const observedRender = useMemo(() => {
+      const blockUpdate = { value: false }
+      const update = () => {
+        // TODO: Decide whether the check for unmount is needed here
+        // Force update unless update is blocked. It's important to block
+        // updates caused by rendering
+        // (when the sync rendening is in progress)
+        if (!blockUpdate.value) forceUpdate()
+      }
+      const trappedRender = trapRender({ render: BaseComponent, blockUpdate, cache, reactionRef })
+      return observe(trappedRender, {
+        scheduler: update,
+        lazy: true
+      })
+    }, [random])
+
+    if (reactionRef.current !== observedRender) reactionRef.current = observedRender
+
+    // clean up observer on unmount
+    useUnmount(() => {
+      // TODO: this does not execute the same amount of times as observe() does,
+      //       probably because of throw's of the async hooks.
+      //       So there probably are memory leaks here. Research this.
+      if (observedRender.current) {
+        unobserve(observedRender.current)
+        observedRender.current = undefined
+      }
+    })
+
+    return observedRender(...args)
+  }
+
+  if (forwardRef) Component = _forwardRef(Component)
+  pipeComponentMeta(BaseComponent, Component)
+
+  Component.__observerOptions = options
+
+  return Component
+}
+
+const DEFAULT_OPTIONS = {
+  forwardRef: false,
+  suspenseProps: {
+    fallback: el(NullComponent, null, null)
+  }
+}
+
+function NullComponent () {
+  return null
+}
+
+function useForceUpdate (throttle) {
+  const [, setTick] = useState()
+  if (throttle) {
+    const timeout = typeof (throttle) === 'number' ? +throttle : DEFAULT_THROTTLE_TIMEOUT
+    // eslint-disable-next-line react-hooks/rules-of-hooks
+    return useCallback(
+      _throttle(() => {
+        setTick(Math.random())
+      }, timeout)
+      , [])
+  } else {
+    return () => setTick(Math.random())
+  }
+}

package/react/executionContextTracker.js

@@ -0,0 +1,32 @@
+export default new class ExecutionContextTracker {
+  #contextId
+  #hooksCounter
+
+  isActive () {
+    return this.#contextId !== undefined
+  }
+
+  getComponentId () {
+    return this.#contextId
+  }
+
+  newHookId () {
+    this.incrementHooksCounter()
+    const id = `_${this.#contextId}_${this.#hooksCounter}`
+    return id
+  }
+
+  incrementHooksCounter () {
+    if (!this.#contextId) return
+    this.#hooksCounter++
+  }
+
+  _start (contextId) {
+    this.#contextId = contextId
+    this.#hooksCounter = -1
+  }
+
+  _clear () {
+    this.#contextId = undefined
+  }
+}()

package/react/helpers.js

@@ -0,0 +1,35 @@
+import { useMemo, useContext, createContext } from 'react'
+import { CACHE_ACTIVE, getDummyCache } from '@startupjs/cache'
+import useIsomorphicLayoutEffect from '@startupjs/utils/useIsomorphicLayoutEffect'
+
+export const ComponentMetaContext = createContext({})
+
+export function pipeComponentDisplayName (SourceComponent, TargetComponent, suffix = '', defaultName = 'StartupjsWrapper') {
+  const displayName = SourceComponent.displayName || SourceComponent.name
+
+  if (!TargetComponent.displayName) {
+    TargetComponent.displayName = displayName ? (displayName + suffix) : defaultName
+  }
+}
+
+export function pipeComponentMeta (SourceComponent, TargetComponent, suffix = '', defaultName = 'StartupjsWrapper') {
+  pipeComponentDisplayName(SourceComponent, TargetComponent, suffix, defaultName)
+
+  if (!TargetComponent.propTypes && SourceComponent.propTypes) {
+    TargetComponent.propTypes = SourceComponent.propTypes
+  }
+  if (!TargetComponent.defaultProps && SourceComponent.defaultProps) {
+    TargetComponent.defaultProps = SourceComponent.defaultProps
+  }
+  return TargetComponent
+}
+
+export function useCache (active) {
+  if (!CACHE_ACTIVE.value || !active) return useMemo(getDummyCache, []) // eslint-disable-line react-hooks/rules-of-hooks
+  const { cache } = useContext(ComponentMetaContext) // eslint-disable-line react-hooks/rules-of-hooks
+  return cache
+}
+
+export function useUnmount (fn) {
+  useIsomorphicLayoutEffect(() => fn, [])
+}

package/react/observer.js

@@ -0,0 +1,9 @@
+import convertToObserver from './convertToObserver.js'
+import wrapIntoSuspense from './wrapIntoSuspense.js'
+
+function observer (Component, options) {
+  return wrapIntoSuspense(convertToObserver(Component, options))
+}
+observer.__wrapObserverMeta = wrapIntoSuspense
+observer.__makeObserver = convertToObserver
+export default observer

package/react/trapRender.js

@@ -0,0 +1,45 @@
+// trap render function (functional component) to block observer updates and activate cache
+// during synchronous rendering
+import { useId } from 'react'
+import { unobserve } from '@nx-js/observer-util'
+import executionContextTracker from './executionContextTracker.js'
+
+export default function trapRender ({ render, blockUpdate, cache, reactionRef }) {
+  return (...args) => {
+    const id = useId()
+    executionContextTracker._start(id)
+    blockUpdate.value = true
+    cache.activate()
+    try {
+      // destroyer.reset() // TODO: this one is for any destructuring logic which might be needed
+      // promiseBatcher.reset() // TODO: this is to support useBatch* hooks
+      const res = render(...args)
+      // if (promiseBatcher.isActive()) {
+      //   throw Error('[react-sharedb] useBatch* hooks were used without a closing useBatch() call.')
+      // }
+      blockUpdate.value = false // TODO: might want to just put it into finally block
+      return res
+    } catch (err) {
+      // TODO: this might only be needed only if promise is thrown
+      //       (check if useUnmount in convertToObserver is called if a regular error is thrown)
+      if (reactionRef.current) {
+        unobserve(reactionRef.current)
+        reactionRef.current = undefined
+      }
+
+      if (!err.then) throw err
+      // If the Promise was thrown, we catch it before Suspense does.
+      // And we run destructors for each hook previous to the one
+      // which did throw this Promise.
+      // We have to manually do it since the unmount logic is not working
+      // for components which were terminated by Suspense as a result of
+      // a promise being thrown.
+      // const destroy = destroyer.getDestructor()
+      // throw err.then(destroy)
+      throw err
+    } finally {
+      cache.deactivate()
+      executionContextTracker._clear()
+    }
+  }
+}

package/react/universal$.js

@@ -0,0 +1,9 @@
+import $ from '../orm/$.js'
+import executionContextTracker from './executionContextTracker.js'
+
+// universal versions of $() which work as a plain function or as a react hook
+export default function universal$ ($root, value) {
+  let id
+  if (executionContextTracker.isActive()) id = executionContextTracker.newHookId()
+  return $($root, value, id)
+}

package/react/universalSub.js

@@ -0,0 +1,9 @@
+import sub from '../orm/sub.js'
+import executionContextTracker from './executionContextTracker.js'
+
+// universal versions of sub() which work as a plain function or as a react hook
+export default function universalSub (...args) {
+  const promiseOrSignal = sub(...args)
+  if (executionContextTracker.isActive() && promiseOrSignal.then) throw promiseOrSignal
+  return promiseOrSignal
+}