turtb 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/LICENSE +661 -0
  2. package/bin/turtb.js +259 -0
  3. package/lib/display/h.js +204 -0
  4. package/lib/display/helpers.js +78 -0
  5. package/lib/display/render.js +177 -0
  6. package/lib/display/shapes.js +91 -0
  7. package/lib/turtle/Signer.js +120 -0
  8. package/lib/turtle/Signer.test.js +38 -0
  9. package/lib/turtle/TurtleBranch.js +147 -0
  10. package/lib/turtle/TurtleBranch.test.js +89 -0
  11. package/lib/turtle/TurtleDictionary.js +157 -0
  12. package/lib/turtle/TurtleDictionary.test.js +331 -0
  13. package/lib/turtle/U8aTurtle.js +203 -0
  14. package/lib/turtle/U8aTurtle.test.js +60 -0
  15. package/lib/turtle/Workspace.js +62 -0
  16. package/lib/turtle/Workspace.test.js +63 -0
  17. package/lib/turtle/codecs/CodecType.js +36 -0
  18. package/lib/turtle/codecs/CodecTypeVersion.js +37 -0
  19. package/lib/turtle/codecs/Commit.js +10 -0
  20. package/lib/turtle/codecs/CompositeCodec.js +86 -0
  21. package/lib/turtle/codecs/TreeNode.js +38 -0
  22. package/lib/turtle/codecs/codec.js +441 -0
  23. package/lib/turtle/connections/AbstractUpdater.js +176 -0
  24. package/lib/turtle/connections/TurtleBranchMultiplexer.js +102 -0
  25. package/lib/turtle/connections/TurtleBranchMultiplexer.test.js +26 -0
  26. package/lib/turtle/connections/TurtleBranchUpdater.js +47 -0
  27. package/lib/turtle/connections/TurtleDB.js +165 -0
  28. package/lib/turtle/connections/TurtleDB.test.js +45 -0
  29. package/lib/turtle/connections/TurtleTalker.js +34 -0
  30. package/lib/turtle/connections/TurtleTalker.test.js +101 -0
  31. package/lib/turtle/utils.js +192 -0
  32. package/lib/turtle/utils.test.js +158 -0
  33. package/lib/utils/Assert.js +115 -0
  34. package/lib/utils/NestedSet.js +68 -0
  35. package/lib/utils/NestedSet.test.js +30 -0
  36. package/lib/utils/OWN_KEYS.js +1 -0
  37. package/lib/utils/Recaller.js +175 -0
  38. package/lib/utils/Recaller.test.js +75 -0
  39. package/lib/utils/TestRunner.js +200 -0
  40. package/lib/utils/TestRunner.test.js +144 -0
  41. package/lib/utils/TestRunnerConstants.js +13 -0
  42. package/lib/utils/combineUint8ArrayLikes.js +18 -0
  43. package/lib/utils/combineUint8Arrays.js +23 -0
  44. package/lib/utils/components.js +88 -0
  45. package/lib/utils/crypto.js +17 -0
  46. package/lib/utils/deepEqual.js +16 -0
  47. package/lib/utils/deepEqual.test.js +27 -0
  48. package/lib/utils/fileTransformer.js +16 -0
  49. package/lib/utils/handleRedirect.js +93 -0
  50. package/lib/utils/logger.js +24 -0
  51. package/lib/utils/nextTick.js +47 -0
  52. package/lib/utils/noble-secp256k1.js +602 -0
  53. package/lib/utils/proxyWithRecaller.js +51 -0
  54. package/lib/utils/toCombinedVersion.js +14 -0
  55. package/lib/utils/toSubVersions.js +14 -0
  56. package/lib/utils/toVersionCount.js +5 -0
  57. package/lib/utils/webSocketMuxFactory.js +123 -0
  58. package/lib/utils/zabacaba.js +25 -0
  59. package/package.json +24 -0
  60. package/src/ArchiveUpdater.js +99 -0
  61. package/src/S3Updater.js +99 -0
  62. package/src/archiveSync.js +28 -0
  63. package/src/fileSync.js +155 -0
  64. package/src/getExistenceLength.js +19 -0
  65. package/src/manageCert.js +36 -0
  66. package/src/originSync.js +75 -0
  67. package/src/outletSync.js +50 -0
  68. package/src/proxyFolder.js +195 -0
  69. package/src/s3Sync.js +32 -0
  70. package/src/webSync.js +101 -0
@@ -0,0 +1,5 @@
1
+ /**
2
+ * @param {Array.<number>} versionArrayCounts
3
+ * @return number
4
+ */
5
+ export const toVersionCount = versionArrayCounts => versionArrayCounts.reduce((acc, value) => acc * value, 1)
@@ -0,0 +1,123 @@
1
+ /* global location, WebSocket */
2
+ import { TurtleBranchMultiplexer } from '../turtle/connections/TurtleBranchMultiplexer.js'
3
+ import { logError, logInfo, logWarn } from './logger.js'
4
+
5
+ /**
6
+ * @typedef {import('../turtle/connections/TurtleDB.js').TurtleDB} TurtleDB
7
+ * @typedef {import('./Recaller.js').Recaller} Recaller
8
+ * @typedef {import('../turtle/connections/TurtleDB.js').TurtleBranchStatus} TurtleBranchStatus
9
+ */
10
+
11
+ const allServiceWorkers = new Set()
12
+
13
+ /**
14
+ * @param {TurtleDB} turtleDB
15
+ * @param {(tbMux: TurtleBranchMultiplexer) => void} callback
16
+ * @param {Recaller} [recaller=turtleDB.recaller]
17
+ */
18
+ export async function webSocketMuxFactory (turtleDB, callback, recaller = turtleDB.recaller) {
19
+ try {
20
+ const serviceWorkerRegistration = await navigator.serviceWorker.register(
21
+ '/service-worker.js',
22
+ { type: 'module', scope: '/' }
23
+ )
24
+ logInfo(() => console.log(' ^^^^^^^ register complete', serviceWorkerRegistration))
25
+ serviceWorkerRegistration.addEventListener('updatefound', () => {
26
+ logInfo(() => console.log(' ^^^^^^^ service-worker update found'))
27
+ })
28
+ try {
29
+ logInfo(() => console.log(' ^^^^^^^ serviceWorkerRegistration.update()'))
30
+ await serviceWorkerRegistration.update()
31
+ } catch (err) {
32
+ logInfo(() => console.log(' ^^^^^^^ serviceWorkerRegistration.update() failed', err))
33
+ }
34
+ logInfo(() => console.log(' ^^^^^^^ serviceWorkerRegistration.update() complete'))
35
+ const { serviceWorker } = navigator
36
+ if (!serviceWorker || allServiceWorkers.has(serviceWorker)) throw new Error('no serviceWorker')
37
+ const { active } = await serviceWorker.ready
38
+ allServiceWorkers.add(serviceWorker)
39
+ const tbMux = new TurtleBranchMultiplexer('serviceWorker', false, turtleDB, recaller)
40
+ const tbMuxBinding = async (/** @type {TurtleBranchStatus} */ status) => {
41
+ logInfo(() => console.log(' ^^^^^^^ tbMuxBinding about to get next'))
42
+ const updater = await tbMux.getTurtleBranchUpdater(tbMux.name, status.publicKey, status.turtleBranch)
43
+ logInfo(() => console.log('updater about to await settle', updater.name))
44
+ await updater.settle
45
+ logInfo(() => console.log('updater settled'))
46
+ }
47
+ turtleDB.bind(tbMuxBinding)
48
+ serviceWorker.onmessage = event => {
49
+ tbMux.incomingBranch.append(new Uint8Array(event.data))
50
+ }
51
+ serviceWorker.startMessages()
52
+ callback?.(tbMux)
53
+ for await (const u8aTurtle of tbMux.outgoingBranch.u8aTurtleGenerator()) {
54
+ active.postMessage(u8aTurtle.uint8Array.buffer)
55
+ }
56
+ } catch (error) {
57
+ logError(() => console.error(error))
58
+ }
59
+
60
+ logWarn(() => console.warn(' ^^^^^^^ unable to connect through service-worker, trying direct websocket connection'))
61
+
62
+ withoutServiceWorker(turtleDB, callback)
63
+ }
64
+
65
+ export async function withoutServiceWorker (turtleDB, callback) {
66
+ const url = `wss://${location.host}`
67
+ let t = 100
68
+ let connectionCount = 0
69
+ while (true) {
70
+ logInfo(() => console.log(' ^^^^^^^ creating new websocket and mux'))
71
+ const tbMux = new TurtleBranchMultiplexer(`backup_websocket_#${connectionCount}`, false, turtleDB)
72
+ for (const publicKey of turtleDB.getPublicKeys()) {
73
+ await tbMux.getTurtleBranchUpdater(tbMux.name, publicKey)
74
+ }
75
+ const tbMuxBinding = async (/** @type {TurtleBranchStatus} */ status) => {
76
+ // logInfo(() => console.log(' ^^^^^^^ tbMuxBinding about to get next', { publicKey }))
77
+ const updater = await tbMux.getTurtleBranchUpdater(tbMux.name, status.publicKey, status.turtleBranch)
78
+ logInfo(() => console.log('updater about to await settle', updater.name))
79
+ await updater.settle
80
+ logInfo(() => console.log('updater settled'))
81
+ }
82
+ turtleDB.bind(tbMuxBinding)
83
+ let connectionIndex
84
+ try {
85
+ connectionIndex = ++connectionCount
86
+ const ws = new WebSocket(url)
87
+ callback?.(tbMux)
88
+ ws.binaryType = 'arraybuffer'
89
+ ws.onopen = async () => {
90
+ logInfo(() => console.log(' ^^^^^^^ onopen, connectionIndex:', connectionIndex))
91
+ ;(async () => {
92
+ try {
93
+ for await (const u8aTurtle of tbMux.outgoingBranch.u8aTurtleGenerator()) {
94
+ if (ws.readyState !== ws.OPEN) break
95
+ ws.send(u8aTurtle.uint8Array)
96
+ }
97
+ } catch (error) {
98
+ logError(() => console.error(error))
99
+ }
100
+ })()
101
+ t = 100
102
+ }
103
+ ws.onmessage = event => {
104
+ if (event.data.byteLength) tbMux.incomingBranch.append(new Uint8Array(event.data))
105
+ else logInfo(() => console.log(`-- keep-alive @ ${(new Date()).toISOString()}`))
106
+ }
107
+ await new Promise((resolve, reject) => {
108
+ ws.onclose = resolve
109
+ ws.onerror = reject
110
+ })
111
+ } catch (error) {
112
+ logError(() => console.error(error))
113
+ }
114
+ tbMux.stop()
115
+ callback?.() // delete old tbMux
116
+ turtleDB.unbind(tbMuxBinding)
117
+ t = Math.min(t, 2 * 60 * 1000) // 2 minutes max (unjittered)
118
+ t = t * (1 + Math.random()) // exponential backoff and some jitter
119
+ logInfo(() => console.log(` ^^^^^^^ waiting ${(t / 1000).toFixed(2)} s`))
120
+ await new Promise(resolve => setTimeout(resolve, t))
121
+ logInfo(() => console.log(' ^^^^^^^ reconnecting...', { connectionIndex }))
122
+ }
123
+ }
@@ -0,0 +1,25 @@
1
+ /**
2
+ * --------------------------------------------------------------------------
3
+ * zabacabadabacabaeabacabadabacabafabacabadabacabaeabacabadabacabagabacabada
4
+ *
5
+ * |
6
+ * | |
7
+ * | | | |
8
+ * | | | | | | | | |
9
+ * | | | | | | | | | | | | | | | | | |
10
+ * | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
11
+ * |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
12
+ * zabacabadabacabaeabacabadabacabafabacabadabacabaeabacabadabacabagabacabada
13
+ * --------------------------------------------------------------------------
14
+ * like the "ruler function" (abacaba) but with numbers for binary-tree-like jumping
15
+ * @param {number} i
16
+ * @returns {number}
17
+ */
18
+
19
+ export function zabacaba (i) {
20
+ if (i === 0) return 0
21
+ if (i === 1) return 1
22
+ const j = ~(i - 1)
23
+ const b = Math.clz32(j & -j) // 31 - b is right zeros
24
+ return 32 - b
25
+ }
package/package.json ADDED
@@ -0,0 +1,24 @@
1
+ {
2
+ "name": "turtb",
3
+ "version": "0.5.0",
4
+ "description": "binary for turtledb projects",
5
+ "repository": "git@github.com:dtudury/turtb.git",
6
+ "author": "David Tudury <david.tudury@gmail.com>",
7
+ "license": "AGPL-3.0-only",
8
+ "bin": "./bin/turtb.js",
9
+ "type": "module",
10
+ "dependencies": {
11
+ "@aws-sdk/client-s3": "^3.958.0",
12
+ "@gerhobbelt/gitignore-parser": "^0.2.0-9",
13
+ "@parcel/watcher": "^2.5.1",
14
+ "commander": "^14.0.2",
15
+ "dotenv": "^17.2.3",
16
+ "express": "^5.2.1",
17
+ "mkcert": "^3.2.0",
18
+ "readline-sync": "^1.4.10",
19
+ "ws": "^8.18.3"
20
+ },
21
+ "scripts": {
22
+ "start": "./bin/turtb.js --env-file .env"
23
+ }
24
+ }
@@ -0,0 +1,99 @@
1
+ import { join } from 'path'
2
+ import { AbstractUpdater } from '../lib/turtle/connections/AbstractUpdater.js'
3
+ import { getExistenceLength } from './getExistenceLength.js'
4
+ import { access, mkdir, readFile, writeFile } from 'fs/promises'
5
+ import { verifyCommitU8a } from '../lib/turtle/Signer.js'
6
+
7
+ /**
8
+ * @typedef {import('@aws-sdk/client-s3').S3Client} S3Client
9
+ * @typedef {import('../lib/utils/Recaller.js').Recaller} Recaller
10
+ */
11
+
12
+ export class ArchiveUpdater extends AbstractUpdater {
13
+ #length
14
+ #lengthPromise
15
+ #getPromises = []
16
+
17
+ /**
18
+ * @param {string} name
19
+ * @param {string} publicKey
20
+ * @param {Recaller} recaller
21
+ * @param {string} path
22
+ */
23
+ constructor (name, publicKey, recaller, path) {
24
+ super(name, publicKey, false, recaller)
25
+ this.path = path
26
+ }
27
+
28
+ indexToPath (index) {
29
+ if (index === undefined) return join(this.path, this.publicKey)
30
+ return join(this.path, this.publicKey, index.toString().padStart(6, '0'))
31
+ }
32
+
33
+ async setUint8ArraysLength (length) {
34
+ const conflictMessage = `Attempt to setUint8ArrayLength(${length}) of "${this.name}" (conflict at ${length.toString().padStart(6, '0')}). Backup and delete archive/${this.publicKey} to resolve.`
35
+ throw new Error(conflictMessage)
36
+ }
37
+
38
+ /**
39
+ * @returns {Promise.<number>}
40
+ */
41
+ async getUint8ArraysLength () {
42
+ if (this.#length !== undefined) return this.#length
43
+ const getExists = async index => {
44
+ try {
45
+ await access(this.indexToPath(index))
46
+ return true
47
+ } catch {
48
+ return false
49
+ }
50
+ }
51
+ if (!this.#lengthPromise) {
52
+ const path = join(this.path, this.publicKey)
53
+ this.#lengthPromise = (async () => {
54
+ if (!(await getExists())) {
55
+ await mkdir(path, { recursive: true })
56
+ }
57
+ this.#length = await getExistenceLength(getExists)
58
+ return this.#length
59
+ })()
60
+ }
61
+ return this.#lengthPromise
62
+ }
63
+
64
+ /**
65
+ * @param {number} index
66
+ * @returns {Promise.<Uint8Array>}
67
+ */
68
+ async getUint8Array (index) {
69
+ await this.getUint8ArraysLength()
70
+ if (index >= this.#length) return
71
+ for (let i = index; i < this.#length; ++i) {
72
+ if (this.#getPromises[i]) break
73
+ this.#getPromises[i] = readFile(this.indexToPath(i))
74
+ }
75
+ return this.#getPromises[index]
76
+ }
77
+
78
+ /**
79
+ * @param {Uint8Array} uint8Array
80
+ * @returns {Promise.<number>}
81
+ */
82
+ async pushUint8Array (uint8Array) {
83
+ await this.getUint8ArraysLength()
84
+ if (!this.#getPromises[this.#length]) {
85
+ this.#getPromises[this.#length] = (async () => {
86
+ let previousUint8Array
87
+ if (this.#length > 0) {
88
+ previousUint8Array = await this.getUint8Array(this.#length - 1)
89
+ }
90
+ const verified = await verifyCommitU8a(this.publicKey, uint8Array, previousUint8Array)
91
+ if (!verified) throw new Error('bad signature')
92
+ await writeFile(this.indexToPath(this.#length), uint8Array, 'binary')
93
+ ++this.#length
94
+ return uint8Array
95
+ })()
96
+ }
97
+ return this.#getPromises[this.#length]
98
+ }
99
+ }
@@ -0,0 +1,99 @@
1
+ import { GetObjectCommand, ListObjectsV2Command, PutObjectCommand } from '@aws-sdk/client-s3'
2
+ import { AbstractUpdater } from '../lib/turtle/connections/AbstractUpdater.js'
3
+ import { verifyCommitU8a } from '../lib/turtle/Signer.js'
4
+ import { getExistenceLength } from './getExistenceLength.js'
5
+
6
+ /**
7
+ * @typedef {import('@aws-sdk/client-s3').S3Client} S3Client
8
+ * @typedef {import('../lib/utils/Recaller.js').Recaller} Recaller
9
+ */
10
+
11
+ export class S3Updater extends AbstractUpdater {
12
+ #length
13
+ #lengthPromise
14
+ #getPromises = []
15
+ /**
16
+ * @param {string} name
17
+ * @param {string} publicKey
18
+ * @param {Recaller} recaller
19
+ * @param {S3Client} s3Client
20
+ * @param {string} bucket
21
+ */
22
+ constructor (name, publicKey, recaller, s3Client, bucket) {
23
+ super(name, publicKey, true, recaller)
24
+ this.s3Client = s3Client
25
+ this.bucket = bucket
26
+ this.indexToKey = (publicKey, index) => `${publicKey}/${index.toString().padStart(6, '0')}`
27
+ }
28
+
29
+ /**
30
+ * @returns {Promise.<number>}
31
+ */
32
+ async getUint8ArraysLength () {
33
+ if (this.#length !== undefined) return this.#length
34
+ const getExists = async index => {
35
+ const listObjectsV2Response = await this.s3Client.send(new ListObjectsV2Command({
36
+ ...(index ? { StartAfter: this.indexToKey(this.publicKey, index - 1) } : {}),
37
+ MaxKeys: 1,
38
+ Bucket: this.bucket,
39
+ Prefix: this.publicKey
40
+ }))
41
+ return !!listObjectsV2Response.KeyCount
42
+ }
43
+ if (this.#lengthPromise === undefined) {
44
+ this.#lengthPromise = getExistenceLength(getExists)
45
+ this.#lengthPromise.then(length => { this.#length = length })
46
+ }
47
+ return this.#lengthPromise
48
+ }
49
+
50
+ /**
51
+ * @param {number} index
52
+ * @returns {Promise.<Uint8Array>}
53
+ */
54
+ async getUint8Array (index) {
55
+ await this.getUint8ArraysLength()
56
+ if (index >= this.#length) return
57
+ for (let i = index; i < this.#length; ++i) {
58
+ if (this.#getPromises[i]) break
59
+ let resolve, reject
60
+ this.#getPromises[i] = new Promise((...args) => { [resolve, reject] = args })
61
+ try {
62
+ const object = await this.s3Client.send(new GetObjectCommand({
63
+ Bucket: this.bucket,
64
+ Key: this.indexToKey(this.publicKey, i)
65
+ }))
66
+ object.Body.transformToByteArray().then(resolve)
67
+ } catch (error) { reject(error) }
68
+ }
69
+ return this.#getPromises[index]
70
+ }
71
+
72
+ /**
73
+ * @param {Uint8Array} uint8Array
74
+ * @returns {Promise.<number>}
75
+ */
76
+ async pushUint8Array (uint8Array) {
77
+ await this.getUint8ArraysLength()
78
+ if (!this.#getPromises[this.#length]) {
79
+ let resolve, reject
80
+ this.#getPromises[this.#length] = new Promise((...args) => { [resolve, reject] = args })
81
+ let previousUint8Array
82
+ if (this.#length > 0) {
83
+ previousUint8Array = await this.getUint8Array(this.#length - 1)
84
+ }
85
+ const verified = await verifyCommitU8a(this.publicKey, uint8Array, previousUint8Array)
86
+ if (!verified) throw new Error('bad signature')
87
+ try {
88
+ await this.s3Client.send(new PutObjectCommand({
89
+ Bucket: this.bucket,
90
+ Body: uint8Array,
91
+ Key: this.indexToKey(this.publicKey, this.#length)
92
+ }))
93
+ ++this.#length
94
+ resolve(uint8Array)
95
+ } catch (error) { reject(error) }
96
+ }
97
+ return this.#getPromises[this.#length]
98
+ }
99
+ }
@@ -0,0 +1,28 @@
1
+ import { TurtleBranchUpdater } from '../lib/turtle/connections/TurtleBranchUpdater.js'
2
+ import { ArchiveUpdater } from './ArchiveUpdater.js'
3
+
4
+ /**
5
+ * @typedef {import('../lib/turtle/connections/TurtleDB.js').TurtleBranchStatus} TurtleBranchStatus
6
+ * @typedef {import('../lib/turtle/connections/TurtleDB.js').TurtleDB} TurtleDB
7
+ * @typedef {import('../lib/utils/Recaller.js').Recaller} Recaller
8
+ */
9
+
10
+ /**
11
+ * @param {TurtleDB} turtleDB
12
+ * @param {Recaller} recaller
13
+ * @param {string} path
14
+ */
15
+ export async function archiveSync (turtleDB, recaller, path) {
16
+ const tbMuxBinding = async (/** @type {TurtleBranchStatus} */ status) => {
17
+ const turtleBranch = status.turtleBranch
18
+ const name = turtleBranch.name
19
+ const publicKey = status.publicKey
20
+ const archiveUpdater = new ArchiveUpdater(`to_archive_#${name}`, publicKey, recaller, path)
21
+ const tbUpdater = new TurtleBranchUpdater(`from_archive_#${name}`, turtleBranch, publicKey, true, recaller)
22
+ archiveUpdater.connect(tbUpdater)
23
+ archiveUpdater.start()
24
+ tbUpdater.start()
25
+ if (!status.bindings.has(tbMuxBinding)) await tbUpdater.settle
26
+ }
27
+ turtleDB.bind(tbMuxBinding)
28
+ }
@@ -0,0 +1,155 @@
1
+ import { join } from 'path'
2
+ import { compile } from '@gerhobbelt/gitignore-parser'
3
+ import { linesToString } from '../lib/utils/fileTransformer.js'
4
+ import { logInfo } from '../lib/utils/logger.js'
5
+ import { THROW } from '../lib/turtle/TurtleDictionary.js'
6
+ import { equalFileObjects, proxyFolder, UPDATES_HANDLER } from './proxyFolder.js'
7
+
8
+ /**
9
+ * @typedef {import('../lib/turtle/connections/TurtleDB.js').TurtleDB} TurtleDB
10
+ * @typedef {import('../lib/turtle/Signer.js').Signer} Signer
11
+ * @typedef {import('../lib/turtle/Workspace.js').Workspace} Workspace
12
+ */
13
+
14
+ const gitFilteredFilenames = (filesObject = {}, turtleDBFolder, gitignoreContent) => {
15
+ gitignoreContent = linesToString(gitignoreContent || filesObject['.gitignore'] || ['.env', '.DS_Store'])
16
+ const gitignore = compile(gitignoreContent)
17
+ const filteredKeys = Object.keys(filesObject).filter(filename => {
18
+ return gitignore.accepts(filename) && !filename.startsWith(turtleDBFolder + '/')
19
+ })
20
+ // console.log(filteredKeys, turtleDBFolder)
21
+ return filteredKeys.sort()
22
+ }
23
+
24
+ const replicateFiles = (a, b, turtleDBFolder, applyGitFilter) => {
25
+ const aFilenames = applyGitFilter ? gitFilteredFilenames(a, turtleDBFolder) : Object.keys(a)
26
+ let bFilenames = applyGitFilter ? gitFilteredFilenames(b, turtleDBFolder) : Object.keys(b)
27
+ let touched = false
28
+ aFilenames.forEach(key => {
29
+ if (!equalFileObjects(b[key], a[key], key)) {
30
+ // console.log(b[key], a[key])
31
+ console.log(`setting b[${key}], replace:${!!b[key]}`)
32
+ b[key] = a[key]
33
+ touched = true
34
+ }
35
+ })
36
+ bFilenames = applyGitFilter ? gitFilteredFilenames(b, turtleDBFolder) : Object.keys(b) // after possible .gitignore update
37
+ bFilenames.forEach(key => {
38
+ if (!aFilenames.includes(key)) {
39
+ console.log(`deleting b[${key}]`)
40
+ delete b[key]
41
+ touched = true
42
+ }
43
+ })
44
+ return touched
45
+ }
46
+
47
+ const syncModule = (turtleBranch, moduleFolder, folderFilesObject, turtleDBFolder) => {
48
+ // return
49
+ const setModuleFiles = moduleFilesObject => {
50
+ const folderFilesObjectCopy = {}
51
+ Object.keys(folderFilesObject)
52
+ .filter(filename => !filename.startsWith(moduleFolder))
53
+ .forEach(filename => {
54
+ folderFilesObjectCopy[filename] = folderFilesObject[filename]
55
+ })
56
+ Object.keys(moduleFilesObject).forEach(filename => {
57
+ folderFilesObjectCopy[join(moduleFolder, filename)] = moduleFilesObject[filename]
58
+ })
59
+ replicateFiles(folderFilesObjectCopy, folderFilesObject, turtleDBFolder, false)
60
+ }
61
+ const turtleWatcher = async () => {
62
+ const moduleFilesObject = turtleBranch.lookup('document', 'value')
63
+ if (!moduleFilesObject || typeof moduleFilesObject !== 'object') throw new Error(`value described in synced folder ${moduleFolder} is not a module object`)
64
+ setModuleFiles(moduleFilesObject)
65
+ }
66
+ turtleBranch.recaller.watch(`fileSync"${turtleBranch.name}"`, turtleWatcher)
67
+ return () => {
68
+ turtleBranch.recaller.unwatch(turtleWatcher)
69
+ }
70
+ }
71
+
72
+ /**
73
+ * @param {string} name
74
+ * @param {TurtleDB} turtleDB
75
+ * @param {Signer} signer
76
+ * @param {string} folder
77
+ * @param {string} resolve
78
+ * @param {string} turtleDBFolder
79
+ * @returns {Promise<Workspace>}
80
+ */
81
+ export async function fileSync (name, turtleDB, signer, folder = '.', resolve = THROW, turtleDBFolder = '.turtleDB') {
82
+ const workspace = await turtleDB.makeWorkspace(signer, name)
83
+ const workspaceFilesObject = workspace.lookup('document', 'value') || {}
84
+ const folderFilesObject = proxyFolder(folder, turtleDB.recaller)
85
+
86
+ // initialize between folder and workspace
87
+ if (gitFilteredFilenames(folderFilesObject).length) {
88
+ if (Object.keys(workspaceFilesObject).length) {
89
+ // clobber for now
90
+ logInfo(() => console.log('clobber filesystem files for now'))
91
+ replicateFiles(workspaceFilesObject, folderFilesObject, turtleDBFolder, true)
92
+ } else {
93
+ logInfo(() => console.log('empty workspace, replicating files to workspace'))
94
+ replicateFiles(folderFilesObject, workspaceFilesObject, turtleDBFolder, true)
95
+ await workspace.commit(workspaceFilesObject, 'initial commit of local tracked files')
96
+ }
97
+ } else {
98
+ if (Object.keys(workspaceFilesObject).length) {
99
+ replicateFiles(workspaceFilesObject, folderFilesObject, turtleDBFolder, true)
100
+ } else {
101
+ throw new Error('TODO: handle first time user')
102
+ }
103
+ }
104
+
105
+ const syncModulesBySymlink = {}
106
+ const addSymlink = symlink => {
107
+ if (!syncModulesBySymlink[symlink]) {
108
+ let unsync = async () => {
109
+ unsync = null
110
+ }
111
+ (async () => {
112
+ const publicKey = symlink.match(/\/([0-9a-z]{40,50})$/)?.[1]
113
+ const turtleBranch = await turtleDB.summonBoundTurtleBranch(publicKey)
114
+ unsync &&= await syncModule(turtleBranch, symlink, folderFilesObject, turtleDBFolder) // &&= in case it got cancelled before the summon completed
115
+ })()
116
+ syncModulesBySymlink[symlink] = { count: 0, unsync }
117
+ }
118
+ ++syncModulesBySymlink[symlink].count
119
+ }
120
+ const removeSymlink = symlink => {
121
+ if (!syncModulesBySymlink[symlink]) throw new Error(`unexpected old symlink "${symlink}" not being tracked`)
122
+ const { count, unsync } = syncModulesBySymlink[symlink] || {}
123
+ if (count > 0) {
124
+ --syncModulesBySymlink[symlink].count
125
+ } else {
126
+ unsync?.()
127
+ delete syncModulesBySymlink[symlink]
128
+ }
129
+ }
130
+ const folderUpdatesHandler = async (changes) => {
131
+ const changesFilenames = gitFilteredFilenames(changes, turtleDBFolder, folderFilesObject['.gitignore'])
132
+ if (!changesFilenames.length) return
133
+ changesFilenames.forEach((filename) => {
134
+ const { oldValue, newValue } = changes[filename]
135
+ if (!equalFileObjects(oldValue, workspaceFilesObject[filename], filename)) throw new Error(`old file value mismatch "${filename}" (TODO: handle collision case)`) // TODO: handle collision case
136
+ workspaceFilesObject[filename] = newValue
137
+ const oldSymlink = oldValue?.symlink
138
+ const newSymlink = newValue?.symlink
139
+ if (oldSymlink === newSymlink) return
140
+ if (oldSymlink) removeSymlink(oldSymlink)
141
+ if (newSymlink) addSymlink(newSymlink)
142
+ })
143
+ await workspace.commit(workspaceFilesObject, 'changes from filesystem')
144
+ }
145
+
146
+ // setup symlinks
147
+ gitFilteredFilenames(folderFilesObject).forEach(filename => {
148
+ const symlink = folderFilesObject[filename].symlink
149
+ if (symlink) addSymlink(symlink)
150
+ })
151
+
152
+ folderFilesObject[UPDATES_HANDLER] = folderUpdatesHandler
153
+
154
+ return workspace
155
+ }
@@ -0,0 +1,19 @@
1
+ export async function getExistenceLength (getExists) {
2
+ let lengthGuess = 0
3
+ if (await getExists(0)) {
4
+ let p = 0
5
+ while (await getExists(2 ** p)) ++p
6
+ if (p < 2) {
7
+ lengthGuess = 2 ** p
8
+ } else {
9
+ lengthGuess = 2 ** (p - 1)
10
+ let direction = 1
11
+ for (let q = p - 2; q >= 0; --q) {
12
+ lengthGuess += direction * 2 ** q
13
+ direction = await getExists(lengthGuess) ? 1 : -1
14
+ }
15
+ if (direction === 1) ++lengthGuess
16
+ }
17
+ }
18
+ return lengthGuess
19
+ }
@@ -0,0 +1,36 @@
1
+ import { dirname } from 'path'
2
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'
3
+ import { createCA, createCert } from 'mkcert'
4
+ import { logError } from '../lib/utils/logger.js'
5
+
6
+ export async function manageCert (fullcertpath) {
7
+ try {
8
+ mkdirSync(dirname(fullcertpath), { recursive: true })
9
+ if (existsSync(fullcertpath)) {
10
+ return JSON.parse(
11
+ readFileSync(fullcertpath, {
12
+ encoding: 'utf8'
13
+ })
14
+ )
15
+ }
16
+ } catch (error) {
17
+ logError(() => console.error(error))
18
+ }
19
+ const ca = await createCA({
20
+ organization: 'TURTLES, Turtles, turtles, etc.',
21
+ countryCode: 'US',
22
+ state: 'California',
23
+ locality: 'Danville',
24
+ validity: 365
25
+ })
26
+ const { key, cert } = await createCert({
27
+ ca: {
28
+ key: ca.key,
29
+ cert: ca.cert
30
+ },
31
+ domains: ['127.0.0.1', 'localhost'],
32
+ validity: 365
33
+ })
34
+ writeFileSync(fullcertpath, JSON.stringify({ key, cert }, null, 4))
35
+ return { key, cert }
36
+ }