turtb 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/bin/turtb.js +259 -0
- package/lib/display/h.js +204 -0
- package/lib/display/helpers.js +78 -0
- package/lib/display/render.js +177 -0
- package/lib/display/shapes.js +91 -0
- package/lib/turtle/Signer.js +120 -0
- package/lib/turtle/Signer.test.js +38 -0
- package/lib/turtle/TurtleBranch.js +147 -0
- package/lib/turtle/TurtleBranch.test.js +89 -0
- package/lib/turtle/TurtleDictionary.js +157 -0
- package/lib/turtle/TurtleDictionary.test.js +331 -0
- package/lib/turtle/U8aTurtle.js +203 -0
- package/lib/turtle/U8aTurtle.test.js +60 -0
- package/lib/turtle/Workspace.js +62 -0
- package/lib/turtle/Workspace.test.js +63 -0
- package/lib/turtle/codecs/CodecType.js +36 -0
- package/lib/turtle/codecs/CodecTypeVersion.js +37 -0
- package/lib/turtle/codecs/Commit.js +10 -0
- package/lib/turtle/codecs/CompositeCodec.js +86 -0
- package/lib/turtle/codecs/TreeNode.js +38 -0
- package/lib/turtle/codecs/codec.js +441 -0
- package/lib/turtle/connections/AbstractUpdater.js +176 -0
- package/lib/turtle/connections/TurtleBranchMultiplexer.js +102 -0
- package/lib/turtle/connections/TurtleBranchMultiplexer.test.js +26 -0
- package/lib/turtle/connections/TurtleBranchUpdater.js +47 -0
- package/lib/turtle/connections/TurtleDB.js +165 -0
- package/lib/turtle/connections/TurtleDB.test.js +45 -0
- package/lib/turtle/connections/TurtleTalker.js +34 -0
- package/lib/turtle/connections/TurtleTalker.test.js +101 -0
- package/lib/turtle/utils.js +192 -0
- package/lib/turtle/utils.test.js +158 -0
- package/lib/utils/Assert.js +115 -0
- package/lib/utils/NestedSet.js +68 -0
- package/lib/utils/NestedSet.test.js +30 -0
- package/lib/utils/OWN_KEYS.js +1 -0
- package/lib/utils/Recaller.js +175 -0
- package/lib/utils/Recaller.test.js +75 -0
- package/lib/utils/TestRunner.js +200 -0
- package/lib/utils/TestRunner.test.js +144 -0
- package/lib/utils/TestRunnerConstants.js +13 -0
- package/lib/utils/combineUint8ArrayLikes.js +18 -0
- package/lib/utils/combineUint8Arrays.js +23 -0
- package/lib/utils/components.js +88 -0
- package/lib/utils/crypto.js +17 -0
- package/lib/utils/deepEqual.js +16 -0
- package/lib/utils/deepEqual.test.js +27 -0
- package/lib/utils/fileTransformer.js +16 -0
- package/lib/utils/handleRedirect.js +93 -0
- package/lib/utils/logger.js +24 -0
- package/lib/utils/nextTick.js +47 -0
- package/lib/utils/noble-secp256k1.js +602 -0
- package/lib/utils/proxyWithRecaller.js +51 -0
- package/lib/utils/toCombinedVersion.js +14 -0
- package/lib/utils/toSubVersions.js +14 -0
- package/lib/utils/toVersionCount.js +5 -0
- package/lib/utils/webSocketMuxFactory.js +123 -0
- package/lib/utils/zabacaba.js +25 -0
- package/package.json +24 -0
- package/src/ArchiveUpdater.js +99 -0
- package/src/S3Updater.js +99 -0
- package/src/archiveSync.js +28 -0
- package/src/fileSync.js +155 -0
- package/src/getExistenceLength.js +19 -0
- package/src/manageCert.js +36 -0
- package/src/originSync.js +75 -0
- package/src/outletSync.js +50 -0
- package/src/proxyFolder.js +195 -0
- package/src/s3Sync.js +32 -0
- package/src/webSync.js +101 -0
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/* global location, WebSocket */
|
|
2
|
+
import { TurtleBranchMultiplexer } from '../turtle/connections/TurtleBranchMultiplexer.js'
|
|
3
|
+
import { logError, logInfo, logWarn } from './logger.js'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* @typedef {import('../turtle/connections/TurtleDB.js').TurtleDB} TurtleDB
|
|
7
|
+
* @typedef {import('./Recaller.js').Recaller} Recaller
|
|
8
|
+
* @typedef {import('../turtle/connections/TurtleDB.js').TurtleBranchStatus} TurtleBranchStatus
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const allServiceWorkers = new Set()
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* @param {TurtleDB} turtleDB
|
|
15
|
+
* @param {(tbMux: TurtleBranchMultiplexer) => void} callback
|
|
16
|
+
* @param {Recaller} [recaller=turtleDB.recaller]
|
|
17
|
+
*/
|
|
18
|
+
export async function webSocketMuxFactory (turtleDB, callback, recaller = turtleDB.recaller) {
|
|
19
|
+
try {
|
|
20
|
+
const serviceWorkerRegistration = await navigator.serviceWorker.register(
|
|
21
|
+
'/service-worker.js',
|
|
22
|
+
{ type: 'module', scope: '/' }
|
|
23
|
+
)
|
|
24
|
+
logInfo(() => console.log(' ^^^^^^^ register complete', serviceWorkerRegistration))
|
|
25
|
+
serviceWorkerRegistration.addEventListener('updatefound', () => {
|
|
26
|
+
logInfo(() => console.log(' ^^^^^^^ service-worker update found'))
|
|
27
|
+
})
|
|
28
|
+
try {
|
|
29
|
+
logInfo(() => console.log(' ^^^^^^^ serviceWorkerRegistration.update()'))
|
|
30
|
+
await serviceWorkerRegistration.update()
|
|
31
|
+
} catch (err) {
|
|
32
|
+
logInfo(() => console.log(' ^^^^^^^ serviceWorkerRegistration.update() failed', err))
|
|
33
|
+
}
|
|
34
|
+
logInfo(() => console.log(' ^^^^^^^ serviceWorkerRegistration.update() complete'))
|
|
35
|
+
const { serviceWorker } = navigator
|
|
36
|
+
if (!serviceWorker || allServiceWorkers.has(serviceWorker)) throw new Error('no serviceWorker')
|
|
37
|
+
const { active } = await serviceWorker.ready
|
|
38
|
+
allServiceWorkers.add(serviceWorker)
|
|
39
|
+
const tbMux = new TurtleBranchMultiplexer('serviceWorker', false, turtleDB, recaller)
|
|
40
|
+
const tbMuxBinding = async (/** @type {TurtleBranchStatus} */ status) => {
|
|
41
|
+
logInfo(() => console.log(' ^^^^^^^ tbMuxBinding about to get next'))
|
|
42
|
+
const updater = await tbMux.getTurtleBranchUpdater(tbMux.name, status.publicKey, status.turtleBranch)
|
|
43
|
+
logInfo(() => console.log('updater about to await settle', updater.name))
|
|
44
|
+
await updater.settle
|
|
45
|
+
logInfo(() => console.log('updater settled'))
|
|
46
|
+
}
|
|
47
|
+
turtleDB.bind(tbMuxBinding)
|
|
48
|
+
serviceWorker.onmessage = event => {
|
|
49
|
+
tbMux.incomingBranch.append(new Uint8Array(event.data))
|
|
50
|
+
}
|
|
51
|
+
serviceWorker.startMessages()
|
|
52
|
+
callback?.(tbMux)
|
|
53
|
+
for await (const u8aTurtle of tbMux.outgoingBranch.u8aTurtleGenerator()) {
|
|
54
|
+
active.postMessage(u8aTurtle.uint8Array.buffer)
|
|
55
|
+
}
|
|
56
|
+
} catch (error) {
|
|
57
|
+
logError(() => console.error(error))
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
logWarn(() => console.warn(' ^^^^^^^ unable to connect through service-worker, trying direct websocket connection'))
|
|
61
|
+
|
|
62
|
+
withoutServiceWorker(turtleDB, callback)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export async function withoutServiceWorker (turtleDB, callback) {
|
|
66
|
+
const url = `wss://${location.host}`
|
|
67
|
+
let t = 100
|
|
68
|
+
let connectionCount = 0
|
|
69
|
+
while (true) {
|
|
70
|
+
logInfo(() => console.log(' ^^^^^^^ creating new websocket and mux'))
|
|
71
|
+
const tbMux = new TurtleBranchMultiplexer(`backup_websocket_#${connectionCount}`, false, turtleDB)
|
|
72
|
+
for (const publicKey of turtleDB.getPublicKeys()) {
|
|
73
|
+
await tbMux.getTurtleBranchUpdater(tbMux.name, publicKey)
|
|
74
|
+
}
|
|
75
|
+
const tbMuxBinding = async (/** @type {TurtleBranchStatus} */ status) => {
|
|
76
|
+
// logInfo(() => console.log(' ^^^^^^^ tbMuxBinding about to get next', { publicKey }))
|
|
77
|
+
const updater = await tbMux.getTurtleBranchUpdater(tbMux.name, status.publicKey, status.turtleBranch)
|
|
78
|
+
logInfo(() => console.log('updater about to await settle', updater.name))
|
|
79
|
+
await updater.settle
|
|
80
|
+
logInfo(() => console.log('updater settled'))
|
|
81
|
+
}
|
|
82
|
+
turtleDB.bind(tbMuxBinding)
|
|
83
|
+
let connectionIndex
|
|
84
|
+
try {
|
|
85
|
+
connectionIndex = ++connectionCount
|
|
86
|
+
const ws = new WebSocket(url)
|
|
87
|
+
callback?.(tbMux)
|
|
88
|
+
ws.binaryType = 'arraybuffer'
|
|
89
|
+
ws.onopen = async () => {
|
|
90
|
+
logInfo(() => console.log(' ^^^^^^^ onopen, connectionIndex:', connectionIndex))
|
|
91
|
+
;(async () => {
|
|
92
|
+
try {
|
|
93
|
+
for await (const u8aTurtle of tbMux.outgoingBranch.u8aTurtleGenerator()) {
|
|
94
|
+
if (ws.readyState !== ws.OPEN) break
|
|
95
|
+
ws.send(u8aTurtle.uint8Array)
|
|
96
|
+
}
|
|
97
|
+
} catch (error) {
|
|
98
|
+
logError(() => console.error(error))
|
|
99
|
+
}
|
|
100
|
+
})()
|
|
101
|
+
t = 100
|
|
102
|
+
}
|
|
103
|
+
ws.onmessage = event => {
|
|
104
|
+
if (event.data.byteLength) tbMux.incomingBranch.append(new Uint8Array(event.data))
|
|
105
|
+
else logInfo(() => console.log(`-- keep-alive @ ${(new Date()).toISOString()}`))
|
|
106
|
+
}
|
|
107
|
+
await new Promise((resolve, reject) => {
|
|
108
|
+
ws.onclose = resolve
|
|
109
|
+
ws.onerror = reject
|
|
110
|
+
})
|
|
111
|
+
} catch (error) {
|
|
112
|
+
logError(() => console.error(error))
|
|
113
|
+
}
|
|
114
|
+
tbMux.stop()
|
|
115
|
+
callback?.() // delete old tbMux
|
|
116
|
+
turtleDB.unbind(tbMuxBinding)
|
|
117
|
+
t = Math.min(t, 2 * 60 * 1000) // 2 minutes max (unjittered)
|
|
118
|
+
t = t * (1 + Math.random()) // exponential backoff and some jitter
|
|
119
|
+
logInfo(() => console.log(` ^^^^^^^ waiting ${(t / 1000).toFixed(2)} s`))
|
|
120
|
+
await new Promise(resolve => setTimeout(resolve, t))
|
|
121
|
+
logInfo(() => console.log(' ^^^^^^^ reconnecting...', { connectionIndex }))
|
|
122
|
+
}
|
|
123
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* --------------------------------------------------------------------------
|
|
3
|
+
* zabacabadabacabaeabacabadabacabafabacabadabacabaeabacabadabacabagabacabada
|
|
4
|
+
*
|
|
5
|
+
* |
|
|
6
|
+
* | |
|
|
7
|
+
* | | | |
|
|
8
|
+
* | | | | | | | | |
|
|
9
|
+
* | | | | | | | | | | | | | | | | | |
|
|
10
|
+
* | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
|
|
11
|
+
* |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
12
|
+
* zabacabadabacabaeabacabadabacabafabacabadabacabaeabacabadabacabagabacabada
|
|
13
|
+
* --------------------------------------------------------------------------
|
|
14
|
+
* like the "ruler function" (abacaba) but with numbers for binary-tree-like jumping
|
|
15
|
+
* @param {number} i
|
|
16
|
+
* @returns {number}
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
export function zabacaba (i) {
|
|
20
|
+
if (i === 0) return 0
|
|
21
|
+
if (i === 1) return 1
|
|
22
|
+
const j = ~(i - 1)
|
|
23
|
+
const b = Math.clz32(j & -j) // 31 - b is right zeros
|
|
24
|
+
return 32 - b
|
|
25
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "turtb",
|
|
3
|
+
"version": "0.5.0",
|
|
4
|
+
"description": "binary for turtledb projects",
|
|
5
|
+
"repository": "git@github.com:dtudury/turtb.git",
|
|
6
|
+
"author": "David Tudury <david.tudury@gmail.com>",
|
|
7
|
+
"license": "AGPL-3.0-only",
|
|
8
|
+
"bin": "./bin/turtb.js",
|
|
9
|
+
"type": "module",
|
|
10
|
+
"dependencies": {
|
|
11
|
+
"@aws-sdk/client-s3": "^3.958.0",
|
|
12
|
+
"@gerhobbelt/gitignore-parser": "^0.2.0-9",
|
|
13
|
+
"@parcel/watcher": "^2.5.1",
|
|
14
|
+
"commander": "^14.0.2",
|
|
15
|
+
"dotenv": "^17.2.3",
|
|
16
|
+
"express": "^5.2.1",
|
|
17
|
+
"mkcert": "^3.2.0",
|
|
18
|
+
"readline-sync": "^1.4.10",
|
|
19
|
+
"ws": "^8.18.3"
|
|
20
|
+
},
|
|
21
|
+
"scripts": {
|
|
22
|
+
"start": "./bin/turtb.js --env-file .env"
|
|
23
|
+
}
|
|
24
|
+
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { join } from 'path'
|
|
2
|
+
import { AbstractUpdater } from '../lib/turtle/connections/AbstractUpdater.js'
|
|
3
|
+
import { getExistenceLength } from './getExistenceLength.js'
|
|
4
|
+
import { access, mkdir, readFile, writeFile } from 'fs/promises'
|
|
5
|
+
import { verifyCommitU8a } from '../lib/turtle/Signer.js'
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* @typedef {import('@aws-sdk/client-s3').S3Client} S3Client
|
|
9
|
+
* @typedef {import('../lib/utils/Recaller.js').Recaller} Recaller
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
export class ArchiveUpdater extends AbstractUpdater {
|
|
13
|
+
#length
|
|
14
|
+
#lengthPromise
|
|
15
|
+
#getPromises = []
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* @param {string} name
|
|
19
|
+
* @param {string} publicKey
|
|
20
|
+
* @param {Recaller} recaller
|
|
21
|
+
* @param {string} path
|
|
22
|
+
*/
|
|
23
|
+
constructor (name, publicKey, recaller, path) {
|
|
24
|
+
super(name, publicKey, false, recaller)
|
|
25
|
+
this.path = path
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
indexToPath (index) {
|
|
29
|
+
if (index === undefined) return join(this.path, this.publicKey)
|
|
30
|
+
return join(this.path, this.publicKey, index.toString().padStart(6, '0'))
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async setUint8ArraysLength (length) {
|
|
34
|
+
const conflictMessage = `Attempt to setUint8ArrayLength(${length}) of "${this.name}" (conflict at ${length.toString().padStart(6, '0')}). Backup and delete archive/${this.publicKey} to resolve.`
|
|
35
|
+
throw new Error(conflictMessage)
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* @returns {Promise.<number>}
|
|
40
|
+
*/
|
|
41
|
+
async getUint8ArraysLength () {
|
|
42
|
+
if (this.#length !== undefined) return this.#length
|
|
43
|
+
const getExists = async index => {
|
|
44
|
+
try {
|
|
45
|
+
await access(this.indexToPath(index))
|
|
46
|
+
return true
|
|
47
|
+
} catch {
|
|
48
|
+
return false
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
if (!this.#lengthPromise) {
|
|
52
|
+
const path = join(this.path, this.publicKey)
|
|
53
|
+
this.#lengthPromise = (async () => {
|
|
54
|
+
if (!(await getExists())) {
|
|
55
|
+
await mkdir(path, { recursive: true })
|
|
56
|
+
}
|
|
57
|
+
this.#length = await getExistenceLength(getExists)
|
|
58
|
+
return this.#length
|
|
59
|
+
})()
|
|
60
|
+
}
|
|
61
|
+
return this.#lengthPromise
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* @param {number} index
|
|
66
|
+
* @returns {Promise.<Uint8Array>}
|
|
67
|
+
*/
|
|
68
|
+
async getUint8Array (index) {
|
|
69
|
+
await this.getUint8ArraysLength()
|
|
70
|
+
if (index >= this.#length) return
|
|
71
|
+
for (let i = index; i < this.#length; ++i) {
|
|
72
|
+
if (this.#getPromises[i]) break
|
|
73
|
+
this.#getPromises[i] = readFile(this.indexToPath(i))
|
|
74
|
+
}
|
|
75
|
+
return this.#getPromises[index]
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* @param {Uint8Array} uint8Array
|
|
80
|
+
* @returns {Promise.<number>}
|
|
81
|
+
*/
|
|
82
|
+
async pushUint8Array (uint8Array) {
|
|
83
|
+
await this.getUint8ArraysLength()
|
|
84
|
+
if (!this.#getPromises[this.#length]) {
|
|
85
|
+
this.#getPromises[this.#length] = (async () => {
|
|
86
|
+
let previousUint8Array
|
|
87
|
+
if (this.#length > 0) {
|
|
88
|
+
previousUint8Array = await this.getUint8Array(this.#length - 1)
|
|
89
|
+
}
|
|
90
|
+
const verified = await verifyCommitU8a(this.publicKey, uint8Array, previousUint8Array)
|
|
91
|
+
if (!verified) throw new Error('bad signature')
|
|
92
|
+
await writeFile(this.indexToPath(this.#length), uint8Array, 'binary')
|
|
93
|
+
++this.#length
|
|
94
|
+
return uint8Array
|
|
95
|
+
})()
|
|
96
|
+
}
|
|
97
|
+
return this.#getPromises[this.#length]
|
|
98
|
+
}
|
|
99
|
+
}
|
package/src/S3Updater.js
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { GetObjectCommand, ListObjectsV2Command, PutObjectCommand } from '@aws-sdk/client-s3'
|
|
2
|
+
import { AbstractUpdater } from '../lib/turtle/connections/AbstractUpdater.js'
|
|
3
|
+
import { verifyCommitU8a } from '../lib/turtle/Signer.js'
|
|
4
|
+
import { getExistenceLength } from './getExistenceLength.js'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* @typedef {import('@aws-sdk/client-s3').S3Client} S3Client
|
|
8
|
+
* @typedef {import('../lib/utils/Recaller.js').Recaller} Recaller
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
export class S3Updater extends AbstractUpdater {
|
|
12
|
+
#length
|
|
13
|
+
#lengthPromise
|
|
14
|
+
#getPromises = []
|
|
15
|
+
/**
|
|
16
|
+
* @param {string} name
|
|
17
|
+
* @param {string} publicKey
|
|
18
|
+
* @param {Recaller} recaller
|
|
19
|
+
* @param {S3Client} s3Client
|
|
20
|
+
* @param {string} bucket
|
|
21
|
+
*/
|
|
22
|
+
constructor (name, publicKey, recaller, s3Client, bucket) {
|
|
23
|
+
super(name, publicKey, true, recaller)
|
|
24
|
+
this.s3Client = s3Client
|
|
25
|
+
this.bucket = bucket
|
|
26
|
+
this.indexToKey = (publicKey, index) => `${publicKey}/${index.toString().padStart(6, '0')}`
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* @returns {Promise.<number>}
|
|
31
|
+
*/
|
|
32
|
+
async getUint8ArraysLength () {
|
|
33
|
+
if (this.#length !== undefined) return this.#length
|
|
34
|
+
const getExists = async index => {
|
|
35
|
+
const listObjectsV2Response = await this.s3Client.send(new ListObjectsV2Command({
|
|
36
|
+
...(index ? { StartAfter: this.indexToKey(this.publicKey, index - 1) } : {}),
|
|
37
|
+
MaxKeys: 1,
|
|
38
|
+
Bucket: this.bucket,
|
|
39
|
+
Prefix: this.publicKey
|
|
40
|
+
}))
|
|
41
|
+
return !!listObjectsV2Response.KeyCount
|
|
42
|
+
}
|
|
43
|
+
if (this.#lengthPromise === undefined) {
|
|
44
|
+
this.#lengthPromise = getExistenceLength(getExists)
|
|
45
|
+
this.#lengthPromise.then(length => { this.#length = length })
|
|
46
|
+
}
|
|
47
|
+
return this.#lengthPromise
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* @param {number} index
|
|
52
|
+
* @returns {Promise.<Uint8Array>}
|
|
53
|
+
*/
|
|
54
|
+
async getUint8Array (index) {
|
|
55
|
+
await this.getUint8ArraysLength()
|
|
56
|
+
if (index >= this.#length) return
|
|
57
|
+
for (let i = index; i < this.#length; ++i) {
|
|
58
|
+
if (this.#getPromises[i]) break
|
|
59
|
+
let resolve, reject
|
|
60
|
+
this.#getPromises[i] = new Promise((...args) => { [resolve, reject] = args })
|
|
61
|
+
try {
|
|
62
|
+
const object = await this.s3Client.send(new GetObjectCommand({
|
|
63
|
+
Bucket: this.bucket,
|
|
64
|
+
Key: this.indexToKey(this.publicKey, i)
|
|
65
|
+
}))
|
|
66
|
+
object.Body.transformToByteArray().then(resolve)
|
|
67
|
+
} catch (error) { reject(error) }
|
|
68
|
+
}
|
|
69
|
+
return this.#getPromises[index]
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* @param {Uint8Array} uint8Array
|
|
74
|
+
* @returns {Promise.<number>}
|
|
75
|
+
*/
|
|
76
|
+
async pushUint8Array (uint8Array) {
|
|
77
|
+
await this.getUint8ArraysLength()
|
|
78
|
+
if (!this.#getPromises[this.#length]) {
|
|
79
|
+
let resolve, reject
|
|
80
|
+
this.#getPromises[this.#length] = new Promise((...args) => { [resolve, reject] = args })
|
|
81
|
+
let previousUint8Array
|
|
82
|
+
if (this.#length > 0) {
|
|
83
|
+
previousUint8Array = await this.getUint8Array(this.#length - 1)
|
|
84
|
+
}
|
|
85
|
+
const verified = await verifyCommitU8a(this.publicKey, uint8Array, previousUint8Array)
|
|
86
|
+
if (!verified) throw new Error('bad signature')
|
|
87
|
+
try {
|
|
88
|
+
await this.s3Client.send(new PutObjectCommand({
|
|
89
|
+
Bucket: this.bucket,
|
|
90
|
+
Body: uint8Array,
|
|
91
|
+
Key: this.indexToKey(this.publicKey, this.#length)
|
|
92
|
+
}))
|
|
93
|
+
++this.#length
|
|
94
|
+
resolve(uint8Array)
|
|
95
|
+
} catch (error) { reject(error) }
|
|
96
|
+
}
|
|
97
|
+
return this.#getPromises[this.#length]
|
|
98
|
+
}
|
|
99
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { TurtleBranchUpdater } from '../lib/turtle/connections/TurtleBranchUpdater.js'
|
|
2
|
+
import { ArchiveUpdater } from './ArchiveUpdater.js'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* @typedef {import('../lib/turtle/connections/TurtleDB.js').TurtleBranchStatus} TurtleBranchStatus
|
|
6
|
+
* @typedef {import('../lib/turtle/connections/TurtleDB.js').TurtleDB} TurtleDB
|
|
7
|
+
* @typedef {import('../lib/utils/Recaller.js').Recaller} Recaller
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* @param {TurtleDB} turtleDB
|
|
12
|
+
* @param {Recaller} recaller
|
|
13
|
+
* @param {string} path
|
|
14
|
+
*/
|
|
15
|
+
export async function archiveSync (turtleDB, recaller, path) {
|
|
16
|
+
const tbMuxBinding = async (/** @type {TurtleBranchStatus} */ status) => {
|
|
17
|
+
const turtleBranch = status.turtleBranch
|
|
18
|
+
const name = turtleBranch.name
|
|
19
|
+
const publicKey = status.publicKey
|
|
20
|
+
const archiveUpdater = new ArchiveUpdater(`to_archive_#${name}`, publicKey, recaller, path)
|
|
21
|
+
const tbUpdater = new TurtleBranchUpdater(`from_archive_#${name}`, turtleBranch, publicKey, true, recaller)
|
|
22
|
+
archiveUpdater.connect(tbUpdater)
|
|
23
|
+
archiveUpdater.start()
|
|
24
|
+
tbUpdater.start()
|
|
25
|
+
if (!status.bindings.has(tbMuxBinding)) await tbUpdater.settle
|
|
26
|
+
}
|
|
27
|
+
turtleDB.bind(tbMuxBinding)
|
|
28
|
+
}
|
package/src/fileSync.js
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { join } from 'path'
|
|
2
|
+
import { compile } from '@gerhobbelt/gitignore-parser'
|
|
3
|
+
import { linesToString } from '../lib/utils/fileTransformer.js'
|
|
4
|
+
import { logInfo } from '../lib/utils/logger.js'
|
|
5
|
+
import { THROW } from '../lib/turtle/TurtleDictionary.js'
|
|
6
|
+
import { equalFileObjects, proxyFolder, UPDATES_HANDLER } from './proxyFolder.js'
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* @typedef {import('../lib/turtle/connections/TurtleDB.js').TurtleDB} TurtleDB
|
|
10
|
+
* @typedef {import('../lib/turtle/Signer.js').Signer} Signer
|
|
11
|
+
* @typedef {import('../lib/turtle/Workspace.js').Workspace} Workspace
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
const gitFilteredFilenames = (filesObject = {}, turtleDBFolder, gitignoreContent) => {
|
|
15
|
+
gitignoreContent = linesToString(gitignoreContent || filesObject['.gitignore'] || ['.env', '.DS_Store'])
|
|
16
|
+
const gitignore = compile(gitignoreContent)
|
|
17
|
+
const filteredKeys = Object.keys(filesObject).filter(filename => {
|
|
18
|
+
return gitignore.accepts(filename) && !filename.startsWith(turtleDBFolder + '/')
|
|
19
|
+
})
|
|
20
|
+
// console.log(filteredKeys, turtleDBFolder)
|
|
21
|
+
return filteredKeys.sort()
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const replicateFiles = (a, b, turtleDBFolder, applyGitFilter) => {
|
|
25
|
+
const aFilenames = applyGitFilter ? gitFilteredFilenames(a, turtleDBFolder) : Object.keys(a)
|
|
26
|
+
let bFilenames = applyGitFilter ? gitFilteredFilenames(b, turtleDBFolder) : Object.keys(b)
|
|
27
|
+
let touched = false
|
|
28
|
+
aFilenames.forEach(key => {
|
|
29
|
+
if (!equalFileObjects(b[key], a[key], key)) {
|
|
30
|
+
// console.log(b[key], a[key])
|
|
31
|
+
console.log(`setting b[${key}], replace:${!!b[key]}`)
|
|
32
|
+
b[key] = a[key]
|
|
33
|
+
touched = true
|
|
34
|
+
}
|
|
35
|
+
})
|
|
36
|
+
bFilenames = applyGitFilter ? gitFilteredFilenames(b, turtleDBFolder) : Object.keys(b) // after possible .gitignore update
|
|
37
|
+
bFilenames.forEach(key => {
|
|
38
|
+
if (!aFilenames.includes(key)) {
|
|
39
|
+
console.log(`deleting b[${key}]`)
|
|
40
|
+
delete b[key]
|
|
41
|
+
touched = true
|
|
42
|
+
}
|
|
43
|
+
})
|
|
44
|
+
return touched
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const syncModule = (turtleBranch, moduleFolder, folderFilesObject, turtleDBFolder) => {
|
|
48
|
+
// return
|
|
49
|
+
const setModuleFiles = moduleFilesObject => {
|
|
50
|
+
const folderFilesObjectCopy = {}
|
|
51
|
+
Object.keys(folderFilesObject)
|
|
52
|
+
.filter(filename => !filename.startsWith(moduleFolder))
|
|
53
|
+
.forEach(filename => {
|
|
54
|
+
folderFilesObjectCopy[filename] = folderFilesObject[filename]
|
|
55
|
+
})
|
|
56
|
+
Object.keys(moduleFilesObject).forEach(filename => {
|
|
57
|
+
folderFilesObjectCopy[join(moduleFolder, filename)] = moduleFilesObject[filename]
|
|
58
|
+
})
|
|
59
|
+
replicateFiles(folderFilesObjectCopy, folderFilesObject, turtleDBFolder, false)
|
|
60
|
+
}
|
|
61
|
+
const turtleWatcher = async () => {
|
|
62
|
+
const moduleFilesObject = turtleBranch.lookup('document', 'value')
|
|
63
|
+
if (!moduleFilesObject || typeof moduleFilesObject !== 'object') throw new Error(`value described in synced folder ${moduleFolder} is not a module object`)
|
|
64
|
+
setModuleFiles(moduleFilesObject)
|
|
65
|
+
}
|
|
66
|
+
turtleBranch.recaller.watch(`fileSync"${turtleBranch.name}"`, turtleWatcher)
|
|
67
|
+
return () => {
|
|
68
|
+
turtleBranch.recaller.unwatch(turtleWatcher)
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* @param {string} name
|
|
74
|
+
* @param {TurtleDB} turtleDB
|
|
75
|
+
* @param {Signer} signer
|
|
76
|
+
* @param {string} folder
|
|
77
|
+
* @param {string} resolve
|
|
78
|
+
* @param {string} turtleDBFolder
|
|
79
|
+
* @returns {Promise<Workspace>}
|
|
80
|
+
*/
|
|
81
|
+
export async function fileSync (name, turtleDB, signer, folder = '.', resolve = THROW, turtleDBFolder = '.turtleDB') {
|
|
82
|
+
const workspace = await turtleDB.makeWorkspace(signer, name)
|
|
83
|
+
const workspaceFilesObject = workspace.lookup('document', 'value') || {}
|
|
84
|
+
const folderFilesObject = proxyFolder(folder, turtleDB.recaller)
|
|
85
|
+
|
|
86
|
+
// initialize between folder and workspace
|
|
87
|
+
if (gitFilteredFilenames(folderFilesObject).length) {
|
|
88
|
+
if (Object.keys(workspaceFilesObject).length) {
|
|
89
|
+
// clobber for now
|
|
90
|
+
logInfo(() => console.log('clobber filesystem files for now'))
|
|
91
|
+
replicateFiles(workspaceFilesObject, folderFilesObject, turtleDBFolder, true)
|
|
92
|
+
} else {
|
|
93
|
+
logInfo(() => console.log('empty workspace, replicating files to workspace'))
|
|
94
|
+
replicateFiles(folderFilesObject, workspaceFilesObject, turtleDBFolder, true)
|
|
95
|
+
await workspace.commit(workspaceFilesObject, 'initial commit of local tracked files')
|
|
96
|
+
}
|
|
97
|
+
} else {
|
|
98
|
+
if (Object.keys(workspaceFilesObject).length) {
|
|
99
|
+
replicateFiles(workspaceFilesObject, folderFilesObject, turtleDBFolder, true)
|
|
100
|
+
} else {
|
|
101
|
+
throw new Error('TODO: handle first time user')
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const syncModulesBySymlink = {}
|
|
106
|
+
const addSymlink = symlink => {
|
|
107
|
+
if (!syncModulesBySymlink[symlink]) {
|
|
108
|
+
let unsync = async () => {
|
|
109
|
+
unsync = null
|
|
110
|
+
}
|
|
111
|
+
(async () => {
|
|
112
|
+
const publicKey = symlink.match(/\/([0-9a-z]{40,50})$/)?.[1]
|
|
113
|
+
const turtleBranch = await turtleDB.summonBoundTurtleBranch(publicKey)
|
|
114
|
+
unsync &&= await syncModule(turtleBranch, symlink, folderFilesObject, turtleDBFolder) // &&= in case it got cancelled before the summon completed
|
|
115
|
+
})()
|
|
116
|
+
syncModulesBySymlink[symlink] = { count: 0, unsync }
|
|
117
|
+
}
|
|
118
|
+
++syncModulesBySymlink[symlink].count
|
|
119
|
+
}
|
|
120
|
+
const removeSymlink = symlink => {
|
|
121
|
+
if (!syncModulesBySymlink[symlink]) throw new Error(`unexpected old symlink "${symlink}" not being tracked`)
|
|
122
|
+
const { count, unsync } = syncModulesBySymlink[symlink] || {}
|
|
123
|
+
if (count > 0) {
|
|
124
|
+
--syncModulesBySymlink[symlink].count
|
|
125
|
+
} else {
|
|
126
|
+
unsync?.()
|
|
127
|
+
delete syncModulesBySymlink[symlink]
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
const folderUpdatesHandler = async (changes) => {
|
|
131
|
+
const changesFilenames = gitFilteredFilenames(changes, turtleDBFolder, folderFilesObject['.gitignore'])
|
|
132
|
+
if (!changesFilenames.length) return
|
|
133
|
+
changesFilenames.forEach((filename) => {
|
|
134
|
+
const { oldValue, newValue } = changes[filename]
|
|
135
|
+
if (!equalFileObjects(oldValue, workspaceFilesObject[filename], filename)) throw new Error(`old file value mismatch "${filename}" (TODO: handle collision case)`) // TODO: handle collision case
|
|
136
|
+
workspaceFilesObject[filename] = newValue
|
|
137
|
+
const oldSymlink = oldValue?.symlink
|
|
138
|
+
const newSymlink = newValue?.symlink
|
|
139
|
+
if (oldSymlink === newSymlink) return
|
|
140
|
+
if (oldSymlink) removeSymlink(oldSymlink)
|
|
141
|
+
if (newSymlink) addSymlink(newSymlink)
|
|
142
|
+
})
|
|
143
|
+
await workspace.commit(workspaceFilesObject, 'changes from filesystem')
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// setup symlinks
|
|
147
|
+
gitFilteredFilenames(folderFilesObject).forEach(filename => {
|
|
148
|
+
const symlink = folderFilesObject[filename].symlink
|
|
149
|
+
if (symlink) addSymlink(symlink)
|
|
150
|
+
})
|
|
151
|
+
|
|
152
|
+
folderFilesObject[UPDATES_HANDLER] = folderUpdatesHandler
|
|
153
|
+
|
|
154
|
+
return workspace
|
|
155
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export async function getExistenceLength (getExists) {
|
|
2
|
+
let lengthGuess = 0
|
|
3
|
+
if (await getExists(0)) {
|
|
4
|
+
let p = 0
|
|
5
|
+
while (await getExists(2 ** p)) ++p
|
|
6
|
+
if (p < 2) {
|
|
7
|
+
lengthGuess = 2 ** p
|
|
8
|
+
} else {
|
|
9
|
+
lengthGuess = 2 ** (p - 1)
|
|
10
|
+
let direction = 1
|
|
11
|
+
for (let q = p - 2; q >= 0; --q) {
|
|
12
|
+
lengthGuess += direction * 2 ** q
|
|
13
|
+
direction = await getExists(lengthGuess) ? 1 : -1
|
|
14
|
+
}
|
|
15
|
+
if (direction === 1) ++lengthGuess
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
return lengthGuess
|
|
19
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { dirname } from 'path'
|
|
2
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'
|
|
3
|
+
import { createCA, createCert } from 'mkcert'
|
|
4
|
+
import { logError } from '../lib/utils/logger.js'
|
|
5
|
+
|
|
6
|
+
export async function manageCert (fullcertpath) {
|
|
7
|
+
try {
|
|
8
|
+
mkdirSync(dirname(fullcertpath), { recursive: true })
|
|
9
|
+
if (existsSync(fullcertpath)) {
|
|
10
|
+
return JSON.parse(
|
|
11
|
+
readFileSync(fullcertpath, {
|
|
12
|
+
encoding: 'utf8'
|
|
13
|
+
})
|
|
14
|
+
)
|
|
15
|
+
}
|
|
16
|
+
} catch (error) {
|
|
17
|
+
logError(() => console.error(error))
|
|
18
|
+
}
|
|
19
|
+
const ca = await createCA({
|
|
20
|
+
organization: 'TURTLES, Turtles, turtles, etc.',
|
|
21
|
+
countryCode: 'US',
|
|
22
|
+
state: 'California',
|
|
23
|
+
locality: 'Danville',
|
|
24
|
+
validity: 365
|
|
25
|
+
})
|
|
26
|
+
const { key, cert } = await createCert({
|
|
27
|
+
ca: {
|
|
28
|
+
key: ca.key,
|
|
29
|
+
cert: ca.cert
|
|
30
|
+
},
|
|
31
|
+
domains: ['127.0.0.1', 'localhost'],
|
|
32
|
+
validity: 365
|
|
33
|
+
})
|
|
34
|
+
writeFileSync(fullcertpath, JSON.stringify({ key, cert }, null, 4))
|
|
35
|
+
return { key, cert }
|
|
36
|
+
}
|