watch-and-mirror 1.0.0

package/README.md

@@ -0,0 +1,29 @@
+# Watch and Mirror
+
+Watch and mirror from and to Hyperdrives and local drives. Enables continuously syncing a remote drive to a local one using peer-to-peer connections.
+
+Built on [Hyperdrive](github.com/holepunchto/hyperdrive), [Mirror Drive](https://github.com/holepunchto/mirror-drive) and [Watch Drive](https://github.com/holepunchto/watch-drive).
+
+## Install
+
+```
+npm i -g watch-and-mirror
+```
+
+## Usage
+
+Run `local-to-drive` on the source machine, and run `drive-to-local` on the target machine. One-to-many replication is trivially possible (just run `drive-to-local` on multiple machines).
+
+Both commands accept an optional environment variable `WATCH_AND_MIRROR_STORAGE_LOC` with the location of the corestore (by default they are stored in `~/.watch-and-mirror`)
+
+```
+drive-to-local <drive key> <destination directory>
+```
+
+```
+local-to-drive <source directory> <drive name>
+```
+
+Where the specified drive name is passed to the corestore namespace (determining the hyperdrive key). The corresponding hyperdrive key will be logged.
+
+Pipe the logs into `pino-pretty` for a human-readable format.

package/drive-to-local.js

@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+
+const process = require('process')
+const Corestore = require('corestore')
+const os = require('os')
+const path = require('path')
+const Hyperswarm = require('hyperswarm')
+const Hyperdrive = require('hyperdrive')
+const fsProm = require('fs').promises
+const IdEnc = require('hypercore-id-encoding')
+const Localdrive = require('localdrive')
+const goodbye = require('graceful-goodbye')
+const pino = require('pino')
+
+const WatchAndMirror = require('.')
+
+const DEFAULT_STORAGE_LOC = path.join(os.homedir(), '.watch-and-mirror', 'drive-to-local-corestore')
+
+function loadConfig () {
+  const res = {
+    bootstrap: undefined,
+    storageLoc: path.resolve(process.env.WATCH_AND_MIRROR_STORAGE_LOC || DEFAULT_STORAGE_LOC)
+  }
+
+  if (process.env.BOOTSTRAP_PORT) { // tests only
+    res.bootstrap = [`127.0.0.1:${process.env.BOOTSTRAP_PORT}`]
+  }
+
+  return res
+}
+
+async function main () {
+  const logger = pino({ name: 'drive-to-local' })
+
+  if (process.argv.length < 4) {
+    logger.error('call with positional arguments <source-hyperdrive-key> <destination-folder-location>')
+    process.exit(1)
+  }
+
+  const sourceKey = IdEnc.normalize(process.argv[2])
+  const destFolder = path.resolve(process.argv[3])
+  await fsProm.mkdir(destFolder, { recursive: true })
+  logger.info(`Mirroring from hyperdrive ${sourceKey} to folder ${destFolder}`)
+
+  const { bootstrap, storageLoc } = loadConfig()
+
+  logger.info(`Using storage ${storageLoc}`)
+  const store = new Corestore(storageLoc)
+
+  const swarm = new Hyperswarm({ bootstrap })
+  swarm.on('connection', (conn, peerInfo) => {
+    const pubKey = IdEnc.normalize(peerInfo.publicKey)
+    const remote = `${conn.rawStream.remoteHost}:${conn.rawStream.remotePort}`
+    logger.info(`Opened connection to peer ${pubKey} (${remote})`)
+
+    conn.on('close', () => {
+      logger.info(`Closed connection to peer ${pubKey} (${remote})`)
+    })
+
+    store.replicate(conn)
+  })
+
+  let watcher = null
+
+  goodbye(async () => {
+    logger.info('shutting down')
+    if (watcher) await watcher.close()
+    await swarm.destroy()
+    await store.close()
+  })
+
+  const sourceDrive = new Hyperdrive(store.session(), sourceKey)
+  await sourceDrive.ready()
+  swarm.join(sourceDrive.discoveryKey)
+
+  const destDrive = new Localdrive(destFolder)
+  await destDrive.ready()
+
+  watcher = new WatchAndMirror(sourceDrive, destDrive)
+  watcher.on('mirrored', ({ add, remove, change }) => {
+    logger.info(`Added ${add}, removed ${remove} and changed ${change} files`)
+  })
+  watcher.on('update-start', diff => {
+    logger.info('Started processing update:')
+    for (const { type, key } of diff) {
+      logger.info(`${type} ${key}`)
+    }
+  })
+  watcher.on('update-done', () => {
+    logger.info('Finished processing update')
+  })
+
+  logger.info('Starting watcher')
+  watcher.ready()
+  logger.info('Running initial sync')
+  await watcher.mirror() // initial sync
+}
+
+main()

package/index.js

@@ -0,0 +1,45 @@
+const { EventEmitter } = require('events')
+const MirrorDrive = require('mirror-drive')
+const watch = require('watch-drive')
+
+class WatchAndMirror extends EventEmitter {
+  constructor (src, dest) {
+    super()
+
+    this.src = src
+    this.dest = dest
+    this._watcher = null
+
+    this.mirror = this._mirrorUndebounced.bind(this)
+  }
+
+  async _mirrorUndebounced () {
+    const mirror = new MirrorDrive(this.src, this.dest)
+    await mirror.done()
+    this.emit('mirrored', mirror.count)
+  }
+
+  ready () {
+    this._watcher = watch(this.src)
+    this._consumeWatcherInBg()
+  }
+
+  async _consumeWatcherInBg () { // Does not throw
+    try {
+      for await (const { diff } of this._watcher) {
+        this.emit('update-start', diff)
+        await this.mirror()
+        this.emit('update-done')
+      }
+    } catch (e) {
+      if (e.message.includes('Stream was destroyed')) return
+      this.emit('error', e)
+    }
+  }
+
+  async close () {
+    await this._watcher.destroy()
+  }
+}
+
+module.exports = WatchAndMirror

package/local-to-drive.js

@@ -0,0 +1,98 @@
+#!/usr/bin/env node
+
+const process = require('process')
+const Corestore = require('corestore')
+const os = require('os')
+const path = require('path')
+const Hyperswarm = require('hyperswarm')
+const Hyperdrive = require('hyperdrive')
+const IdEnc = require('hypercore-id-encoding')
+const Localdrive = require('localdrive')
+const goodbye = require('graceful-goodbye')
+const pino = require('pino')
+
+const WatchAndMirror = require('.')
+
+const DEFAULT_STORAGE_LOC = path.join(os.homedir(), '.watch-and-mirror', 'local-to-drive-corestore')
+
+function loadConfig () {
+  const res = {
+    bootstrap: undefined,
+    storageLoc: path.resolve(process.env.WATCH_AND_MIRROR_STORAGE_LOC || DEFAULT_STORAGE_LOC)
+  }
+
+  if (process.env.BOOTSTRAP_PORT) { // tests only
+    res.bootstrap = [`127.0.0.1:${process.env.BOOTSTRAP_PORT}`]
+  }
+
+  return res
+}
+
+async function main () {
+  const logger = pino({ name: 'local-to-drive' })
+
+  if (process.argv.length < 4) {
+    logger.error('call with positional arguments <source-folder-location> <target-hyperdrive-name')
+    process.exit(1)
+  }
+
+  const sourceFolder = path.resolve(process.argv[2])
+  const destDriveName = process.argv[3]
+
+  const { bootstrap, storageLoc } = loadConfig()
+
+  logger.info(`Using storage ${storageLoc}`)
+  const store = new Corestore(storageLoc)
+
+  const swarm = new Hyperswarm({ bootstrap })
+  swarm.on('connection', (conn, peerInfo) => {
+    const pubKey = IdEnc.normalize(peerInfo.publicKey)
+    const remote = `${conn.rawStream.remoteHost}:${conn.rawStream.remotePort}`
+    logger.info(`Opened connection to peer ${pubKey} (${remote})`)
+
+    conn.on('close', () => {
+      logger.info(`Closed connection to peer ${pubKey} (${remote})`)
+    })
+
+    store.replicate(conn)
+  })
+
+  let watcher = null
+
+  goodbye(async () => {
+    logger.info('shutting down')
+    if (watcher) await watcher.close()
+    await swarm.destroy()
+    await store.close()
+  })
+
+  const sourceDrive = new Localdrive(sourceFolder)
+  await sourceDrive.ready()
+
+  const destDrive = new Hyperdrive(store.namespace(destDriveName))
+  await destDrive.ready()
+  swarm.join(destDrive.discoveryKey)
+
+  logger.info(`Mirroring from folder ${sourceFolder} to hyperdrive ${IdEnc.normalize(destDrive.key)} (identified in corestore by name '${destDriveName}')`)
+
+  watcher = new WatchAndMirror(sourceDrive, destDrive)
+  watcher.on('mirrored', ({ add, remove, change }) => {
+    logger.info(`Added ${add}, removed ${remove} and changed ${change} files`)
+  })
+  watcher.on('update-start', diff => {
+    logger.info('Started processing update:')
+    for (const { type, key } of diff) {
+      logger.info(`${type} ${key}`)
+    }
+  })
+  watcher.on('update-done', () => {
+    logger.info('Finished processing update')
+  })
+
+  logger.info('Starting watcher')
+  watcher.ready()
+  logger.info('Running initial sync')
+  await watcher.mirror() // initial sync
+}
+
+main()

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "watch-and-mirror",
+  "version": "1.0.0",
+  "description": "Watch and mirror from and to Hyperdrives and local drives",
+  "main": "index.js",
+  "scripts": {
+    "test": "standard && brittle test.js"
+  },
+  "bin": {
+    "drive-to-local": "./drive-to-local.js",
+    "local-to-drive": "./local-to-drive.js"
+  },
+  "files": [
+    "index.js",
+    "drive-to-local.js",
+    "local-to-drive.js"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@gitlab.com/dcent-tech/watch-and-mirror.git"
+  },
+  "keywords": [
+    "watch",
+    "drive",
+    "mirror",
+    "hyperdrive",
+    "peer",
+    "to",
+    "peer"
+  ],
+  "author": "H. Degroote",
+  "license": "Apache-2.0",
+  "bugs": {
+    "url": "https://gitlab.com/dcent-tech/watch-and-mirror/issues"
+  },
+  "homepage": "https://gitlab.com/dcent-tech/watch-and-mirror#readme",
+  "devDependencies": {
+    "b4a": "^1.7.3",
+    "brittle": "^3.19.0",
+    "newline-decoder": "^1.0.2",
+    "standard": "^17.1.2"
+  },
+  "dependencies": {
+    "corestore": "^7.4.8",
+    "graceful-goodbye": "^1.3.3",
+    "hypercore-id-encoding": "^1.3.0",
+    "hyperdrive": "^13.0.1",
+    "hyperswarm": "^4.14.0",
+    "localdrive": "^2.2.0",
+    "mirror-drive": "^1.6.0",
+    "pino": "^10.0.0",
+    "watch-drive": "^2.0.3"
+  }
+}