umap-project 2.6.3__py3-none-any.whl → 2.7.0__py3-none-any.whl

umap/static/umap/js/modules/sync/engine.js

@@ -1,6 +1,46 @@
+import * as Utils from '../utils.js'
+import { HybridLogicalClock } from './hlc.js'
 import { DataLayerUpdater, FeatureUpdater, MapUpdater } from './updaters.js'
 import { WebSocketTransport } from './websocket.js'
 
+/**
+ * The syncEngine exposes an API to sync messages between peers over the network.
+ *
+ * It's taking care of initializing the `transport` layer (sending and receiving
+ * messages over websocket), the `operations` list (to store them locally),
+ * and the `updaters` to apply messages to the map.
+ *
+ * You can use the `update`, `upsert` and `delete` methods.
+ *
+ * @example
+ *
+ * ```
+ * const sync = new SyncEngine(map)
+ *
+ * // Get the authentication token from the umap server
+ * sync.authenticate(tokenURI, webSocketURI, server)
+ *
+ * // Alternatively, start the engine manually with
+ * sync.start(webSocketURI, authToken)
+ *
+ * // Then use the `upsert`, `update` and `delete` methods.
+ * let {metadata, subject} = object.getSyncMetadata()
+ * sync.upsert(subject, metadata, "value")
+ * sync.update(subject, metadata, "key", "value")
+ * sync.delete(subject, metadata, "key")
+ * ```
+ *
+ * A `proxy()` method is also exposed, making it easier to use without having
+ * to specify `subject` and `metadata` fields on each call:
+ *
+ * @example
+ * ```
+ * // Or using the `proxy()` method:
+ * let syncProxy = sync.proxy(object)
+ * syncProxy.upsert("value")
+ * syncProxy.update("key", "value")
+ * ```
+ */
 export class SyncEngine {
   constructor(map) {
     this.updaters = {
@@ -9,6 +49,7 @@ export class SyncEngine {
       datalayer: new DataLayerUpdater(map),
     }
     this.transport = undefined
+    this._operations = new Operations()
   }
 
   async authenticate(tokenURI, webSocketURI, server) {
@@ -27,6 +68,27 @@ export class SyncEngine {
     this.transport = undefined
   }
 
+  upsert(subject, metadata, value) {
+    this._send({ verb: 'upsert', subject, metadata, value })
+  }
+
+  update(subject, metadata, key, value) {
+    this._send({ verb: 'update', subject, metadata, key, value })
+  }
+
+  delete(subject, metadata, key) {
+    this._send({ verb: 'delete', subject, metadata, key })
+  }
+
+  _send(inputMessage) {
+    const message = this._operations.addLocal(inputMessage)
+
+    if (this.offline) return
+    if (this.transport) {
+      this.transport.send('OperationMessage', message)
+    }
+  }
+
   _getUpdater(subject, metadata) {
     if (Object.keys(this.updaters).includes(subject)) {
       return this.updaters[subject]
@@ -34,32 +96,170 @@ export class SyncEngine {
     throw new Error(`Unknown updater ${subject}, ${metadata}`)
   }
 
-  // This method is called by the transport layer on new messages
+  _applyOperation(operation) {
+    const updater = this._getUpdater(operation.subject, operation.metadata)
+    updater.applyMessage(operation)
+  }
+
+  getNumberOfConnectedPeers() {
+    if (this.peers) return this.peers.length
+    return 0
+  }
+
+  /**
+   * This is called by the transport layer on new messages,
+   * and dispatches the different "on*" methods.
+   */
   receive({ kind, ...payload }) {
-    if (kind === 'operation') {
-      const updater = this._getUpdater(payload.subject, payload.metadata)
-      updater.applyMessage(payload)
+    if (kind === 'OperationMessage') {
+      this.onOperationMessage(payload)
+    } else if (kind === 'JoinResponse') {
+      this.onJoinResponse(payload)
+    } else if (kind === 'ListPeersResponse') {
+      this.onListPeersResponse(payload)
+    } else if (kind === 'PeerMessage') {
+      debug('received peermessage', payload)
+      if (payload.message.verb === 'ListOperationsRequest') {
+        this.onListOperationsRequest(payload)
+      } else if (payload.message.verb === 'ListOperationsResponse') {
+        this.onListOperationsResponse(payload)
+      }
     } else {
-      throw new Error(`Unknown dispatch kind: ${kind}`)
+      throw new Error(`Received unknown message from the websocket server: ${kind}`)
     }
   }
 
-  _send(message) {
-    if (this.transport) {
-      this.transport.send('operation', message)
+  /**
+   * Received when an operation has been performed by another peer.
+   *
+   * Stores the passed operation locally and apply it.
+   *
+   * @param {Object} payload
+   */
+  onOperationMessage(payload) {
+    this._operations.storeRemoteOperations([payload])
+    this._applyOperation(payload)
+  }
+
+  /**
+   * Received when the server acknowledges the `join` for this peer.
+   *
+   * @param {Object} payload
+   * @param {string} payload.uuid The server-assigned uuid for this peer
+   * @param {string[]} payload.peers The list of peers uuids
+   */
+  onJoinResponse({ uuid, peers }) {
+    debug('received join response', { uuid, peers })
+    this.uuid = uuid
+    this.onListPeersResponse({ peers })
+
+    // Get one peer at random
+    const randomPeer = this._getRandomPeer()
+
+    if (randomPeer) {
+      // Retrieve the operations which happened before join.
+      this.sendToPeer(randomPeer, 'ListOperationsRequest', {
+        lastKnownHLC: this._operations.getLastKnownHLC(),
+      })
     }
   }
 
-  upsert(subject, metadata, value) {
-    this._send({ verb: 'upsert', subject, metadata, value })
+  /**
+   * Received when the list of peers has changed.
+   *
+   * @param {Object} payload
+   * @param {string[]} payload.peers The list of peers uuids
+   */
+  onListPeersResponse({ peers }) {
+    debug('received peerinfo', { peers })
+    this.peers = peers
+    this.updaters.map.update({ key: 'numberOfConnectedPeers' })
   }
 
-  update(subject, metadata, key, value) {
-    this._send({ verb: 'update', subject, metadata, key, value })
+  /**
+   * Received when another peer asks for the list of operations.
+   *
+   * @param {Object} payload
+   * @param {string} payload.sender the uuid of the requesting peer
+   * @param {string} payload.latestKnownHLC the latest known HLC of the requesting peer
+   */
+  onListOperationsRequest({ sender, lastKnownHLC }) {
+    this.sendToPeer(sender, 'ListOperationsResponse', {
+      operations: this._operations.getOperationsSince(lastKnownHLC),
+    })
   }
 
-  delete(subject, metadata, key) {
-    this._send({ verb: 'delete', subject, metadata, key })
+  /**
+   * Received when another peer sends the list of operations.
+   *
+   * When receiving this message, operations are filtered and applied
+   *
+   * @param {*} operations The list of (encoded operations)
+   */
+  onListOperationsResponse({ sender, message }) {
+    debug(`received operations from peer ${sender}`, message.operations)
+
+    if (message.operations.length === 0) return
+
+    // Get the list of stored operations before this message.
+    const remoteOperations = Operations.sort(message.operations)
+    this._operations.storeRemoteOperations(remoteOperations)
+
+    // Sort the local operations only once, see below.
+    for (const remote of remoteOperations) {
+      if (this._operations.shouldBypassOperation(remote)) {
+        debug(
+          'Skipping the following operation, because a newer one has been found locally',
+          remote
+        )
+      } else {
+        this._applyOperation(remote)
+      }
+    }
+
+    // TODO: compact the changes here?
+    // e.g. we might want to :
+    // - group cases of multiple updates
+    // - not apply changes where we have a more recent version (but store them nevertheless)
+
+    // 1. Get the list of fields that changed (in the incoming operations)
+    // 2. For each field, get the last version
+    // 3. Check if we should apply the changes.
+
+    // For each operation
+    // Get the updated key hlc
+    // If key.local_hlc > key.remote_hlc: drop
+    // Else: apply
+  }
+
+  /**
+   * Send a message to another peer (via the transport layer)
+   *
+   * @param {*} recipient
+   * @param {*} verb
+   * @param {*} payload
+   */
+  sendToPeer(recipient, verb, payload) {
+    payload.verb = verb
+    this.transport.send('PeerMessage', {
+      sender: this.uuid,
+      recipient: recipient,
+      message: payload,
+    })
+  }
+
+  /**
+   * Selects a peer ID at random within the known ones.
+   *
+   * @returns {string|bool} the selected peer uuid, or False if none was found.
+   */
+  _getRandomPeer() {
+    const otherPeers = this.peers.filter((p) => p !== this.uuid)
+    if (otherPeers.length > 0) {
+      const random = Math.floor(Math.random() * otherPeers.length)
+      return otherPeers[random]
+    }
+    return false
   }
 
   /**
@@ -91,3 +291,160 @@ export class SyncEngine {
     return new Proxy(this, handler)
   }
 }
+
+/**
+ * Registry of local and remote operations, keeping a constant ordering.
+ */
+export class Operations {
+  constructor() {
+    this._hlc = new HybridLogicalClock()
+    this._operations = new Array()
+  }
+
+  /**
+   * Tick the clock and add store the passed message in the operations list.
+   *
+   * @param {*} inputMessage
+   * @returns {*} clock-aware message
+   */
+  addLocal(inputMessage) {
+    const message = { ...inputMessage, hlc: this._hlc.tick() }
+    this._operations.push(message)
+    return message
+  }
+
+  /**
+   * Returns the current list of operations ordered by their HLC.
+   *
+   * This DOES NOT modify the list in place, but instead return a new copy.
+   *
+   * @returns {Array}
+   */
+  sorted() {
+    return Operations.sort(this._operations)
+  }
+
+  /**
+   * Static method to order the given list of operations by their HCL.
+   *
+   * @param {Object[]} operations
+   * @returns an ordered copy
+   */
+  static sort(operations) {
+    const copy = [...operations]
+    copy.sort((a, b) => (a.hlc < b.hlc ? -1 : 1))
+    return copy
+  }
+
+  /**
+   * Store a list of remote operations locally
+   *
+   * Note that operations are not applied as part of this method.
+   *
+   * - Updates the list of operations with the remote ones.
+   * - Updates the clock to reflect these changes.
+   *
+   * @param {Array} remoteOperations
+   */
+  storeRemoteOperations(remoteOperations) {
+    // get the highest date from the passed operations
+    const greatestHLC = remoteOperations
+      .map((op) => op.hlc)
+      .reduce((max, current) => (current > max ? current : max))
+
+    // Bump the current HLC.
+    this._hlc.receive(greatestHLC)
+    this._operations.push(...remoteOperations)
+  }
+
+  /**
+   * Get operations that happened since a specific clock tick.
+   */
+  getOperationsSince(hlc) {
+    if (!hlc) return this._operations
+    // first get the position of the clock that was sent
+    const start = this._operations.findIndex((op) => op.hlc === hlc)
+    this._operations.slice(start)
+    return this._operations.filter((op) => op.hlc > hlc)
+  }
+
+  /**
+   * Returns the last known HLC value.
+   */
+  getLastKnownHLC() {
+    return this._operations.at(-1)?.hlc
+  }
+
+  /**
+   * Checks if a given operation should be bypassed.
+   *
+   * Note that this doesn't only check the clock, but also if the operation share
+   * on the same context (subject + metadata).
+   *
+   * @param {Object} remote the remote operation to compare to
+   * @returns bool
+   */
+  shouldBypassOperation(remote) {
+    const sortedLocalOperations = this.sorted()
+    // No operations are stored, no need to check
+    if (sortedLocalOperations.length <= 0) {
+      debug('No operations are stored, no need to check')
+      return false
+    }
+
+    // Latest local operation is older than the remote one
+    const latest = sortedLocalOperations.at(-1)
+    if (latest.hlc < remote.hlc) {
+      debug('Latest local operation is older than the remote one')
+      return false
+    }
+
+    // Skip operations enabling the sync engine:
+    // If we receive something, we are already connected.
+    if (
+      remote.hasOwnProperty('key') &&
+      remote.key === 'options.syncEnabled' &&
+      remote.value === true
+    ) {
+      return true
+    }
+    for (const local of sortedLocalOperations) {
+      if (
+        local.hlc > remote.hlc &&
+        Operations.haveSameContext(local, remote) &&
+        // For now (and until we fix the conflict between updates and upsert)
+        // upsert always have priority over other operations
+        remote.verb !== 'upsert'
+      ) {
+        debug('this is newer:', local)
+        return true
+      }
+    }
+    return false
+  }
+
+  /**
+   * Compares two operations to see if they share the same context.
+   *
+   * @param {Object} local
+   * @param {Object} remote
+   * @return {bool} true if the two operations share the same context.
+   */
+  static haveSameContext(local, remote) {
+    const shouldCheckKey =
+      local.hasOwnProperty('key') &&
+      remote.hasOwnProperty('key') &&
+      typeof local.key !== 'undefined' &&
+      typeof remote.key !== 'undefined'
+
+    return (
+      Utils.deepEqual(local.subject, remote.subject) &&
+      Utils.deepEqual(local.metadata, remote.metadata) &&
+      (!shouldCheckKey || (shouldCheckKey && local.key == remote.key))
+    )
+  }
+}
+
+function debug(...args) {
+  console.debug('SYNC ⇆', ...args)
+}

umap/static/umap/js/modules/sync/hlc.js

@@ -0,0 +1,106 @@
+import * as Utils from '../utils.js'
+
+/**
+ * This is an implementation of a Hybrid Logical Clock (HLC).
+ *
+ * There are three parts in the clock:
+ *
+ * - walltime: the relative clock of each of the peers
+ * - NN: a local counter that gets incremented in case of ties.
+ * - id: to identify the peer
+ *
+ * HLCs are used to order operations consistently in distributed systems.
+ */
+export class HybridLogicalClock {
+  constructor(walltime = Date.now(), nn = 0, id = Utils.generateId()) {
+    this._current = { walltime, nn, id }
+  }
+
+  /**
+   * Return a serialized version of the current clock
+   */
+  serialize(clock = this._current) {
+    const { walltime, nn, id } = clock
+    return `${walltime}:${nn}:${id}`
+  }
+
+  /**
+   * Parse a serialized time and return a JS object.
+   * @param string raw
+   * @returns object
+   */
+  parse(raw) {
+    const tokens = raw.split(':')
+
+    if (tokens.length !== 3) {
+      throw new SyntaxError(`Unable to parse ${raw}`)
+    }
+    const [walltime, rawNN, id] = tokens
+
+    let nn = Number.parseInt(rawNN)
+    if (Number.isNaN(nn)) {
+      nn = 0
+    }
+    return { walltime, nn, id }
+  }
+
+  /**
+   * Increment the current clock by one tick.
+   *
+   * - If the current time is greater than the last known tip, increment it.
+   * - Otherwise, increment the `nn` counter by 1.
+   *
+   * This allows each tick to be different from each other.
+   *
+   * @returns a serialized clock
+   */
+  tick() {
+    // Copy the current value of the hlc to avoid concurrency issues
+    const current = { ...this._current }
+    const now = Date.now()
+
+    let nextValue
+
+    if (now > current.walltime) {
+      nextValue = { ...current, walltime: now, nn: 0 }
+    } else {
+      nextValue = { ...current, nn: current.nn + 1 }
+    }
+
+    this._current = nextValue
+    return this.serialize(this._current)
+  }
+
+  /**
+   * Receive a remote clock info, and update the local clock.
+   *
+   * - If the current wall time is greater than both local and remote wall time, use the local one.
+   * - If the current wall time is the same, increment max (local, remote) `nn` counter by 1.
+   * - If remote time is greater, keep the remote time and increment `nn`
+   * - Otherwise, keep local values and increment `nn`
+   *
+   * This allows to take into account clock drifting, when clocks on different peers are getting
+   * out of sync.
+   **/
+  receive(remoteRaw) {
+    const local = { ...this._current }
+    const remote = this.parse(remoteRaw)
+    const now = Date.now()
+
+    let nextValue
+
+    if (now > local.walltime && now > remote.walltime) {
+      nextValue = { ...local, walltime: now }
+    } else if (local.walltime == remote.walltime) {
+      const nn = Math.max(local.nn, remote.nn) + 1
+      nextValue = { ...local, nn: nn }
+    } else if (remote.walltime > local.walltime) {
+      nextValue = { ...remote, id: local.id, nn: remote.nn + 1 }
+    } else {
+      nextValue = { ...local, nn: local.nn + 1 }
+    }
+
+    this._current = nextValue
+    return this._current
+  }
+}

umap/static/umap/js/modules/sync/updaters.js

@@ -1,6 +1,8 @@
+import { fieldInSchema } from '../utils.js'
+
 /**
- * This file contains the updaters: classes that are able to convert messages
- * received from another party (or the server) to changes on the map.
+ * Updaters are classes able to convert messages
+ * received from other peers (or from the server) to changes on the map.
  */
 
 class BaseUpdater {
@@ -42,7 +44,10 @@ class BaseUpdater {
 
 export class MapUpdater extends BaseUpdater {
   update({ key, value }) {
-    this.updateObjectValue(this.map, key, value)
+    if (fieldInSchema(key)) {
+      this.updateObjectValue(this.map, key, value)
+    }
+
     this.map.render([key])
   }
 }
@@ -56,7 +61,14 @@ export class DataLayerUpdater extends BaseUpdater {
 
   update({ key, metadata, value }) {
     const datalayer = this.getDataLayerFromID(metadata.id)
-    this.updateObjectValue(datalayer, key, value)
+    if (fieldInSchema(key)) {
+      this.updateObjectValue(datalayer, key, value)
+    } else {
+      console.debug(
+        'Not applying update for datalayer because key is not in the schema',
+        key
+      )
+    }
     datalayer.render([key])
   }
 }
@@ -76,7 +88,7 @@ export class FeatureUpdater extends BaseUpdater {
     if (feature) {
       feature.geometry = value.geometry
     } else {
-      datalayer.makeFeature(value)
+      datalayer.makeFeature(value, false)
     }
   }
 
@@ -85,9 +97,9 @@ export class FeatureUpdater extends BaseUpdater {
     const feature = this.getFeatureFromMetadata(metadata)
     if (feature === undefined) {
       console.error(`Unable to find feature with id = ${metadata.id}.`)
+      return
     }
     if (key === 'geometry') {
-      const datalayer = this.getDataLayerFromID(metadata.layerId)
       const feature = this.getFeatureFromMetadata(metadata, value)
       feature.geometry = value
     } else {

umap/static/umap/js/modules/sync/websocket.js

@@ -2,7 +2,7 @@ export class WebSocketTransport {
   constructor(webSocketURI, authToken, messagesReceiver) {
     this.websocket = new WebSocket(webSocketURI)
     this.websocket.onopen = () => {
-      this.send('join', { token: authToken })
+      this.send('JoinRequest', { token: authToken })
     }
     this.websocket.addEventListener('message', this.onMessage.bind(this))
     this.receiver = messagesReceiver

umap/static/umap/js/modules/tableeditor.js

@@ -64,7 +64,7 @@ export default class TableEditor extends WithTemplate {
         action: () => this.deleteProperty(property),
       })
     }
-    this.contextmenu.open([event.clientX, event.clientY], actions)
+    this.contextmenu.open(event, actions)
   }
 
   renderHeaders() {

umap/static/umap/js/modules/ui/base.js

@@ -72,9 +72,9 @@ export class Positioned {
       left = x - this.container.offsetWidth
     }
     if (y < window.innerHeight / 2) {
-      top = y
+      top = Math.min(y, window.innerHeight - this.container.offsetHeight)
     } else {
-      top = y - this.container.offsetHeight
+      top = Math.max(0, y - this.container.offsetHeight)
     }
     this.setPosition({ left, top })
   }

umap/static/umap/js/modules/ui/contextmenu.js

@@ -15,27 +15,60 @@ export default class ContextMenu extends Positioned {
     })
   }
 
-  open([x, y], items) {
+  open(event, items) {
+    const left = event.clientX
+    const top = event.clientY
+    this.openAt([left, top], items)
+  }
+
+  openBelow(element, items) {
+    const coords = this.getPosition(element)
+    this.openAt([coords.left, coords.bottom], items)
+  }
+
+  openAt([left, top], items) {
     this.container.innerHTML = ''
     for (const item of items) {
-      const li = loadTemplate(
-        `<li class="${item.className || ''}"><button tabindex="0" class="flat">${item.label}</button></li>`
-      )
-      li.addEventListener('click', () => {
-        this.close()
-        item.action()
-      })
-      this.container.appendChild(li)
-    }
-    document.body.appendChild(this.container)
-    this.computePosition([x, y])
-    this.container.querySelector('button').focus()
-    this.container.addEventListener('keydown', (event) => {
-      if (event.key === 'Escape') {
-        event.stopPropagation()
-        this.close()
+      if (item === '-') {
+        this.container.appendChild(document.createElement('hr'))
+      } else if (typeof item.action === 'string') {
+        const li = loadTemplate(
+          `<li class="${item.className || ''}"><a tabindex="0" href="${item.action}">${item.label}</a></li>`
+        )
+        this.container.appendChild(li)
+      } else {
+        const li = loadTemplate(
+          `<li class="${item.className || ''}"><button tabindex="0" class="flat">${item.label}</button></li>`
+        )
+        li.addEventListener('click', () => {
+          this.close()
+          item.action()
+        })
+        this.container.appendChild(li)
       }
-    })
+    }
+    // When adding contextmenu below the map container, clicking on any link will send the
+    // "focusout" element on link click, preventing to trigger the click action
+    const parent = document
+      .elementFromPoint(left, top)
+      .closest('.leaflet-container').parentNode
+    parent.appendChild(this.container)
+    if (this.options.fixed) {
+      this.setPosition({ left, top })
+    } else {
+      this.computePosition([left, top])
+    }
+    this.container.querySelector('li > *').focus()
+    this.container.addEventListener(
+      'keydown',
+      (event) => {
+        if (event.key === 'Escape') {
+          event.stopPropagation()
+          this.close()
+        }
+      },
+      { once: true }
+    )
   }
 
   close() {