primitive-admin 1.2.0-alpha.0 → 1.2.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/dist/bin/primitive.js +22 -24
- package/dist/bin/primitive.js.map +1 -1
- package/dist/src/commands/agent-sessions.d.ts +20 -0
- package/dist/src/commands/agent-sessions.js +275 -0
- package/dist/src/commands/agent-sessions.js.map +1 -0
- package/dist/src/commands/apps-children.d.ts +54 -0
- package/dist/src/commands/apps-children.js +578 -0
- package/dist/src/commands/apps-children.js.map +1 -0
- package/dist/src/commands/apps.js +88 -9
- package/dist/src/commands/apps.js.map +1 -1
- package/dist/src/commands/auth-sessions.d.ts +3 -3
- package/dist/src/commands/auth-sessions.js +117 -23
- package/dist/src/commands/auth-sessions.js.map +1 -1
- package/dist/src/commands/auth.js +248 -15
- package/dist/src/commands/auth.js.map +1 -1
- package/dist/src/commands/connections.js +1 -5
- package/dist/src/commands/connections.js.map +1 -1
- package/dist/src/commands/databases.js +7 -109
- package/dist/src/commands/databases.js.map +1 -1
- package/dist/src/commands/documents.d.ts +29 -0
- package/dist/src/commands/documents.js +676 -191
- package/dist/src/commands/documents.js.map +1 -1
- package/dist/src/commands/env.d.ts +9 -1
- package/dist/src/commands/env.js +158 -45
- package/dist/src/commands/env.js.map +1 -1
- package/dist/src/commands/functions.js +7 -1
- package/dist/src/commands/functions.js.map +1 -1
- package/dist/src/commands/init.js +2 -1
- package/dist/src/commands/init.js.map +1 -1
- package/dist/src/commands/integrations.js +14 -2
- package/dist/src/commands/integrations.js.map +1 -1
- package/dist/src/commands/prompts.js +4 -0
- package/dist/src/commands/prompts.js.map +1 -1
- package/dist/src/commands/scripts.js +1 -1
- package/dist/src/commands/scripts.js.map +1 -1
- package/dist/src/commands/sessions.js +3 -8
- package/dist/src/commands/sessions.js.map +1 -1
- package/dist/src/commands/sync-app-settings.d.ts +25 -7
- package/dist/src/commands/sync-app-settings.js +81 -20
- package/dist/src/commands/sync-app-settings.js.map +1 -1
- package/dist/src/commands/sync.d.ts +183 -12
- package/dist/src/commands/sync.js +974 -452
- package/dist/src/commands/sync.js.map +1 -1
- package/dist/src/commands/tokens.js +1 -6
- package/dist/src/commands/tokens.js.map +1 -1
- package/dist/src/commands/users.js +77 -2
- package/dist/src/commands/users.js.map +1 -1
- package/dist/src/lib/api-client.d.ts +161 -35
- package/dist/src/lib/api-client.js +210 -86
- package/dist/src/lib/api-client.js.map +1 -1
- package/dist/src/lib/app-settings-descriptor.d.ts +9 -1
- package/dist/src/lib/app-settings-descriptor.js +22 -10
- package/dist/src/lib/app-settings-descriptor.js.map +1 -1
- package/dist/src/lib/auth-flow.d.ts +41 -1
- package/dist/src/lib/auth-flow.js +45 -16
- package/dist/src/lib/auth-flow.js.map +1 -1
- package/dist/src/lib/child-apps-local.d.ts +184 -0
- package/dist/src/lib/child-apps-local.js +282 -0
- package/dist/src/lib/child-apps-local.js.map +1 -0
- package/dist/src/lib/ci-session-request.d.ts +22 -0
- package/dist/src/lib/ci-session-request.js +51 -0
- package/dist/src/lib/ci-session-request.js.map +1 -0
- package/dist/src/lib/codegen-shared/generatedFiles.d.ts +4 -1
- package/dist/src/lib/codegen-shared/generatedFiles.js +19 -13
- package/dist/src/lib/codegen-shared/generatedFiles.js.map +1 -1
- package/dist/src/lib/config-object-descriptor.js +5 -5
- package/dist/src/lib/config-object-descriptor.js.map +1 -1
- package/dist/src/lib/config-payload.js +2 -3
- package/dist/src/lib/config-payload.js.map +1 -1
- package/dist/src/lib/config-surface.d.ts +2 -1
- package/dist/src/lib/config-surface.js +7 -28
- package/dist/src/lib/config-surface.js.map +1 -1
- package/dist/src/lib/credentials-store.d.ts +17 -2
- package/dist/src/lib/credentials-store.js +72 -6
- package/dist/src/lib/credentials-store.js.map +1 -1
- package/dist/src/lib/document-export-lookups.d.ts +31 -0
- package/dist/src/lib/document-export-lookups.js +53 -0
- package/dist/src/lib/document-export-lookups.js.map +1 -0
- package/dist/src/lib/document-export-permissions.d.ts +7 -5
- package/dist/src/lib/document-export-permissions.js +7 -5
- package/dist/src/lib/document-export-permissions.js.map +1 -1
- package/dist/src/lib/env-resolver-core.d.ts +69 -1
- package/dist/src/lib/env-resolver-core.js +137 -20
- package/dist/src/lib/env-resolver-core.js.map +1 -1
- package/dist/src/lib/env-resolver.d.ts +14 -2
- package/dist/src/lib/env-resolver.js +32 -10
- package/dist/src/lib/env-resolver.js.map +1 -1
- package/dist/src/lib/function-run.d.ts +4 -1
- package/dist/src/lib/function-run.js +24 -4
- package/dist/src/lib/function-run.js.map +1 -1
- package/dist/src/lib/function-sync.d.ts +5 -2
- package/dist/src/lib/function-sync.js +7 -4
- package/dist/src/lib/function-sync.js.map +1 -1
- package/dist/src/lib/generated-config-surfaces.d.ts +49 -50
- package/dist/src/lib/generated-config-surfaces.js +283 -166
- package/dist/src/lib/generated-config-surfaces.js.map +1 -1
- package/dist/src/lib/generated-sdk-types.d.ts +1 -1
- package/dist/src/lib/generated-sdk-types.js +1 -1
- package/dist/src/lib/generated-sdk-types.js.map +1 -1
- package/dist/src/lib/init-adopt.js +12 -3
- package/dist/src/lib/init-adopt.js.map +1 -1
- package/dist/src/lib/local-state.d.ts +17 -1
- package/dist/src/lib/local-state.js +94 -15
- package/dist/src/lib/local-state.js.map +1 -1
- package/dist/src/lib/local-test-cases.d.ts +5 -2
- package/dist/src/lib/local-test-cases.js +7 -4
- package/dist/src/lib/local-test-cases.js.map +1 -1
- package/dist/src/lib/log-inspection.d.ts +25 -0
- package/dist/src/lib/log-inspection.js +24 -0
- package/dist/src/lib/log-inspection.js.map +1 -1
- package/dist/src/lib/logout-admin-session.d.ts +2 -1
- package/dist/src/lib/logout-admin-session.js +7 -1
- package/dist/src/lib/logout-admin-session.js.map +1 -1
- package/dist/src/lib/pull-write.d.ts +31 -0
- package/dist/src/lib/pull-write.js +49 -0
- package/dist/src/lib/pull-write.js.map +1 -0
- package/dist/src/lib/root-import-decision.d.ts +59 -0
- package/dist/src/lib/root-import-decision.js +95 -0
- package/dist/src/lib/root-import-decision.js.map +1 -0
- package/dist/src/lib/scope-request.d.ts +69 -0
- package/dist/src/lib/scope-request.js +188 -0
- package/dist/src/lib/scope-request.js.map +1 -0
- package/dist/src/lib/session-id.d.ts +9 -0
- package/dist/src/lib/session-id.js +28 -0
- package/dist/src/lib/session-id.js.map +1 -0
- package/dist/src/lib/snapshots.d.ts +17 -1
- package/dist/src/lib/snapshots.js +50 -2
- package/dist/src/lib/snapshots.js.map +1 -1
- package/dist/src/lib/sync-paths.d.ts +35 -0
- package/dist/src/lib/sync-paths.js +57 -5
- package/dist/src/lib/sync-paths.js.map +1 -1
- package/dist/src/lib/transient-retry.d.ts +34 -0
- package/dist/src/lib/transient-retry.js +57 -0
- package/dist/src/lib/transient-retry.js.map +1 -0
- package/dist/src/types/index.d.ts +21 -4
- package/package.json +4 -3
|
@@ -10,7 +10,8 @@ import { pageCursorOption, pageLimitOption, parsePageLimit, printEmptyPage, prin
|
|
|
10
10
|
import { normalizeCliListEnvelope, wholeListEnvelope, } from "../lib/paginate.js";
|
|
11
11
|
import { parseDataOption } from "../lib/data-input.js";
|
|
12
12
|
import { parseFilterOptions } from "../lib/record-filter.js";
|
|
13
|
-
import { buildPermissionsExport } from "../lib/document-export-permissions.js";
|
|
13
|
+
import { buildPermissionsExport, PENDING_INVITATIONS_ABSENT_STATUSES, } from "../lib/document-export-permissions.js";
|
|
14
|
+
import { readExportLookup } from "../lib/document-export-lookups.js";
|
|
14
15
|
import { chunkAddressOf, renumberManifestForExport, } from "../lib/snapshot-manifest-layout.js";
|
|
15
16
|
import { validateSnapshotManifest } from "js-bao";
|
|
16
17
|
import { auditSnapshotChain, } from "../lib/snapshot-audit.js";
|
|
@@ -19,9 +20,12 @@ import { createAuditLocalStore, createChainFingerprintReader, } from "../lib/sna
|
|
|
19
20
|
import { discoverIngestInput, } from "../lib/document-ingest-input.js";
|
|
20
21
|
import { buildIngestArtifact, ingestSchemaFromIntrospection, } from "../lib/document-ingest-artifact.js";
|
|
21
22
|
import { driveDocumentIngest, summariseIngestPlan, assertIngestConfirmable, } from "../lib/document-ingest.js";
|
|
23
|
+
import { withTransientRetry } from "../lib/transient-retry.js";
|
|
22
24
|
import { ulid } from "ulid";
|
|
25
|
+
import * as Y from "yjs";
|
|
23
26
|
import * as fs from "fs";
|
|
24
27
|
import * as path from "path";
|
|
28
|
+
import { decideRootImport, rootDocumentFormatOf, } from "../lib/root-import-decision.js";
|
|
25
29
|
/**
|
|
26
30
|
* One bulk-load session as rows (#3434, #3435, #3598).
|
|
27
31
|
*
|
|
@@ -124,6 +128,32 @@ export function describeBulkFailure(err) {
|
|
|
124
128
|
},
|
|
125
129
|
};
|
|
126
130
|
}
|
|
131
|
+
/**
|
|
132
|
+
* The `--data-file` help of `documents records bulk`: the op shapes the route
|
|
133
|
+
* accepts, including the `upsert` action addressed by a declared unique
|
|
134
|
+
* constraint (#3750).
|
|
135
|
+
*/
|
|
136
|
+
export const BULK_DATA_FILE_HELP = "JSON file with { operations: [...] } or a bare operations array. " +
|
|
137
|
+
"Each op is { model, action: create|patch|delete, id, data, precondition? } " +
|
|
138
|
+
"— `data` holds the record fields and is required (non-empty) on " +
|
|
139
|
+
"create/patch, not allowed on delete; a create `id` must be a 26-char " +
|
|
140
|
+
"uppercase Crockford ULID. An upsert op is { model, action: upsert, " +
|
|
141
|
+
"constraint, data, id?, precondition? } — `constraint` names a unique " +
|
|
142
|
+
"constraint the model declares and `data` carries its fields; the record " +
|
|
143
|
+
"holding that key is merged into, or created (at `id` when given)";
|
|
144
|
+
/**
|
|
145
|
+
* The success line of `documents records bulk` (#3750): the blob's counts,
|
|
146
|
+
* with the upsert ops and how many of them created their record. An answer
|
|
147
|
+
* without `upserted` (an older server) reads as no upserts.
|
|
148
|
+
*/
|
|
149
|
+
export function describeBulkResult(result) {
|
|
150
|
+
const upserted = Array.isArray(result.upserted) ? result.upserted : [];
|
|
151
|
+
const created = upserted.filter((entry) => entry.created).length;
|
|
152
|
+
return (`Applied ${result.applied} operation(s): ` +
|
|
153
|
+
`${result.added.length} added, ${result.updated.length} updated, ` +
|
|
154
|
+
`${result.deleted} deleted, ${upserted.length} upserted` +
|
|
155
|
+
`${upserted.length > 0 ? ` (${created} created)` : ""}.`);
|
|
156
|
+
}
|
|
127
157
|
/**
|
|
128
158
|
* A document's tags as a list, from whatever the wire handed back (#3644).
|
|
129
159
|
*
|
|
@@ -1034,13 +1064,9 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1034
1064
|
// Apply an atomic multi-model operations blob
|
|
1035
1065
|
records
|
|
1036
1066
|
.command("bulk")
|
|
1037
|
-
.description("Apply an atomic create/patch/delete operations blob to a document")
|
|
1067
|
+
.description("Apply an atomic create/patch/delete/upsert operations blob to a document")
|
|
1038
1068
|
.argument("<document-id>", "Document ID")
|
|
1039
|
-
.requiredOption("--data-file <file>",
|
|
1040
|
-
"Each op is { model, action: create|patch|delete, id, data, precondition? } " +
|
|
1041
|
-
"— `data` holds the record fields and is required (non-empty) on " +
|
|
1042
|
-
"create/patch, not allowed on delete; a create `id` must be a 26-char " +
|
|
1043
|
-
"uppercase Crockford ULID")
|
|
1069
|
+
.requiredOption("--data-file <file>", BULK_DATA_FILE_HELP)
|
|
1044
1070
|
.option("--app <app-id>", "App ID")
|
|
1045
1071
|
.option(DOCUMENT_FORMAT_FLAG, DOCUMENT_FORMAT_HELP)
|
|
1046
1072
|
.option("-y, --yes", "Skip confirmation prompt")
|
|
@@ -1079,8 +1105,7 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1079
1105
|
json(result);
|
|
1080
1106
|
return;
|
|
1081
1107
|
}
|
|
1082
|
-
success(
|
|
1083
|
-
`${result.added.length} added, ${result.updated.length} updated, ${result.deleted} deleted.`);
|
|
1108
|
+
success(describeBulkResult(result));
|
|
1084
1109
|
}
|
|
1085
1110
|
catch (err) {
|
|
1086
1111
|
const report = describeBulkFailure(err);
|
|
@@ -1135,7 +1160,7 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1135
1160
|
// Document statistics (platform-vocabulary projection)
|
|
1136
1161
|
documents
|
|
1137
1162
|
.command("stats")
|
|
1138
|
-
.description("Show a document's record/model/blob counts and
|
|
1163
|
+
.description("Show a document's record/model/blob counts and size")
|
|
1139
1164
|
.argument("<document-id>", "Document ID")
|
|
1140
1165
|
.option("--app <app-id>", "App ID")
|
|
1141
1166
|
.option("--json", "Output as JSON")
|
|
@@ -1152,7 +1177,11 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1152
1177
|
keyValue("Records", result.recordCount);
|
|
1153
1178
|
keyValue("Models", result.modelCount);
|
|
1154
1179
|
keyValue("Blobs", result.blobCount);
|
|
1155
|
-
|
|
1180
|
+
// A large document's size is its records' stored JSON, exact; an
|
|
1181
|
+
// ordinary document's is an estimate (#4193).
|
|
1182
|
+
keyValue(result.sizeBasis === "records"
|
|
1183
|
+
? "Size (bytes, records)"
|
|
1184
|
+
: "Size (bytes, approx)", result.sizeBytes);
|
|
1156
1185
|
keyValue("Last modified", result.lastModifiedAt ? formatDate(result.lastModifiedAt) : "—");
|
|
1157
1186
|
}
|
|
1158
1187
|
catch (err) {
|
|
@@ -1925,6 +1954,9 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1925
1954
|
const resolvedAppId = resolveAppId(appIdArg, options);
|
|
1926
1955
|
const client = new ApiClient();
|
|
1927
1956
|
const includeBlobs = options.blobs !== false;
|
|
1957
|
+
// A document that fails does not stop the others; the run reports each
|
|
1958
|
+
// one by id and exits non-zero (#4149).
|
|
1959
|
+
const failed = [];
|
|
1928
1960
|
try {
|
|
1929
1961
|
const documents = await client.listAdminDocuments(resolvedAppId, options.userId);
|
|
1930
1962
|
let docList = Array.isArray(documents) ? documents : [];
|
|
@@ -1945,6 +1977,7 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1945
1977
|
}
|
|
1946
1978
|
catch (err) {
|
|
1947
1979
|
warn(`Failed to export ${docId}: ${err.message}`);
|
|
1980
|
+
failed.push({ documentId: docId, error: String(err?.message ?? err) });
|
|
1948
1981
|
}
|
|
1949
1982
|
}
|
|
1950
1983
|
// Write manifest. `export-all` is per-user, and each run REPLACES the
|
|
@@ -1979,16 +2012,25 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1979
2012
|
fs.mkdirSync(path.dirname(manifestPath), { recursive: true });
|
|
1980
2013
|
fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
|
|
1981
2014
|
if (options.json) {
|
|
1982
|
-
json(manifest);
|
|
2015
|
+
json(failed.length > 0 ? { ...manifest, failed } : manifest);
|
|
1983
2016
|
}
|
|
1984
2017
|
else {
|
|
1985
2018
|
success(`Exported ${exportedIds.length} document(s) to ${options.output}`);
|
|
2019
|
+
if (failed.length > 0) {
|
|
2020
|
+
error(`Failed to export ${failed.length} document(s): ` +
|
|
2021
|
+
`${failed.map((entry) => entry.documentId).join(", ")}. ` +
|
|
2022
|
+
`The manifest lists only the documents exported.`);
|
|
2023
|
+
}
|
|
1986
2024
|
}
|
|
1987
2025
|
}
|
|
1988
2026
|
catch (err) {
|
|
1989
2027
|
error(err.message);
|
|
1990
2028
|
process.exit(1);
|
|
1991
2029
|
}
|
|
2030
|
+
if (failed.length > 0) {
|
|
2031
|
+
await flushOutput();
|
|
2032
|
+
process.exit(1);
|
|
2033
|
+
}
|
|
1992
2034
|
});
|
|
1993
2035
|
documents
|
|
1994
2036
|
.command("import")
|
|
@@ -2023,6 +2065,7 @@ Permissions (enforced by the server, not the CLI):
|
|
|
2023
2065
|
process.exit(1);
|
|
2024
2066
|
}
|
|
2025
2067
|
const client = new ApiClient();
|
|
2068
|
+
let failedCount = 0;
|
|
2026
2069
|
// Resolve --owner to a userId through the one shared lookup (#2763).
|
|
2027
2070
|
let ownerUserId;
|
|
2028
2071
|
if (options.owner) {
|
|
@@ -2075,7 +2118,9 @@ Permissions (enforced by the server, not the CLI):
|
|
|
2075
2118
|
blobsUploaded: 0,
|
|
2076
2119
|
aliasesSet: 0,
|
|
2077
2120
|
aliasesSkipped: 0,
|
|
2121
|
+
completed: 0,
|
|
2078
2122
|
rootDocuments: [],
|
|
2123
|
+
failed: [],
|
|
2079
2124
|
};
|
|
2080
2125
|
// Where every root-marked export in this run lands, decided from reads
|
|
2081
2126
|
// alone and before any write (#3135): a refusal, or two exports
|
|
@@ -2083,12 +2128,18 @@ Permissions (enforced by the server, not the CLI):
|
|
|
2083
2128
|
const rootPlans = await planRootImports(client, resolvedAppId, docDirs, options.owner, ownerUserId, options.overwrite || false);
|
|
2084
2129
|
// Nothing is created until every artifact is one this CLI can install:
|
|
2085
2130
|
// a refusal halfway through a multi-document import would leave the
|
|
2086
|
-
// app half-populated (#2816). A root export is
|
|
2087
|
-
//
|
|
2088
|
-
//
|
|
2131
|
+
// app half-populated (#2816). A root export is vouched for only when
|
|
2132
|
+
// its plan installs it (#4195): a refused or skipped root export is
|
|
2133
|
+
// never read past `metadata.json`, so a broken chain in it cannot
|
|
2134
|
+
// fail the run.
|
|
2089
2135
|
for (const docDir of docDirs) {
|
|
2090
|
-
|
|
2136
|
+
const rootPlan = rootPlans.get(docDir);
|
|
2137
|
+
if (rootPlan) {
|
|
2138
|
+
if (rootPlan.mode === "install") {
|
|
2139
|
+
readChainArtifact(docDir, rootPlan.exportedDocumentId);
|
|
2140
|
+
}
|
|
2091
2141
|
continue;
|
|
2142
|
+
}
|
|
2092
2143
|
if (!largeDocumentArtifactAt(docDir))
|
|
2093
2144
|
continue;
|
|
2094
2145
|
const metadataPath = path.join(docDir, "metadata.json");
|
|
@@ -2097,17 +2148,44 @@ Permissions (enforced by the server, not the CLI):
|
|
|
2097
2148
|
: path.basename(docDir);
|
|
2098
2149
|
readChainArtifact(docDir, documentId);
|
|
2099
2150
|
}
|
|
2151
|
+
// One document failing does not stop the run (#4148): it is recorded
|
|
2152
|
+
// by id, the rest are imported, and the run exits non-zero at the end.
|
|
2100
2153
|
for (const docDir of docDirs) {
|
|
2101
|
-
|
|
2154
|
+
const progress = {
|
|
2155
|
+
documentId: readExportMetadata(docDir)?.documentId || path.basename(docDir),
|
|
2156
|
+
exists: false,
|
|
2157
|
+
};
|
|
2158
|
+
try {
|
|
2159
|
+
await importSingleDocument(client, resolvedAppId, docDir, options.overwrite || false, options.aliases || "skip", options.dryRun || false, summary, progress, ownerUserId, rootPlans.get(docDir));
|
|
2160
|
+
}
|
|
2161
|
+
catch (err) {
|
|
2162
|
+
const failure = {
|
|
2163
|
+
documentId: progress.documentId,
|
|
2164
|
+
status: progress.exists ? "incomplete" : "failed",
|
|
2165
|
+
error: err?.message || String(err),
|
|
2166
|
+
};
|
|
2167
|
+
summary.failed.push(failure);
|
|
2168
|
+
error(`Document ${failure.documentId} ` +
|
|
2169
|
+
`${failure.status === "incomplete" ? "is incomplete" : "failed"}: ` +
|
|
2170
|
+
failure.error);
|
|
2171
|
+
}
|
|
2102
2172
|
}
|
|
2173
|
+
failedCount = summary.failed.length;
|
|
2103
2174
|
if (options.json) {
|
|
2104
2175
|
json(summary);
|
|
2105
2176
|
}
|
|
2106
2177
|
else {
|
|
2107
|
-
|
|
2178
|
+
if (failedCount === 0) {
|
|
2179
|
+
success(`Import complete:`);
|
|
2180
|
+
}
|
|
2181
|
+
else {
|
|
2182
|
+
error(`Import finished, ${failedCount} document(s) not imported:`);
|
|
2183
|
+
}
|
|
2108
2184
|
keyValue(" Documents created", String(summary.created));
|
|
2109
2185
|
if (summary.updated > 0)
|
|
2110
2186
|
keyValue(" Documents updated", String(summary.updated));
|
|
2187
|
+
if (summary.completed > 0)
|
|
2188
|
+
keyValue(" Documents completed", String(summary.completed));
|
|
2111
2189
|
if (summary.skipped > 0)
|
|
2112
2190
|
keyValue(" Documents skipped", String(summary.skipped));
|
|
2113
2191
|
if (summary.blobsUploaded > 0)
|
|
@@ -2116,12 +2194,25 @@ Permissions (enforced by the server, not the CLI):
|
|
|
2116
2194
|
keyValue(" Aliases set", String(summary.aliasesSet));
|
|
2117
2195
|
if (summary.aliasesSkipped > 0)
|
|
2118
2196
|
keyValue(" Aliases skipped", String(summary.aliasesSkipped));
|
|
2197
|
+
for (const failure of summary.failed) {
|
|
2198
|
+
keyValue(` ${failure.documentId}`, `${failure.status}: ${failure.error}`);
|
|
2199
|
+
}
|
|
2200
|
+
if (failedCount > 0) {
|
|
2201
|
+
info("Re-run the same import to finish them: a document a run left " +
|
|
2202
|
+
"incomplete is completed, and one already imported is skipped.");
|
|
2203
|
+
}
|
|
2119
2204
|
}
|
|
2120
2205
|
}
|
|
2121
2206
|
catch (err) {
|
|
2122
2207
|
error(err.message);
|
|
2123
2208
|
process.exit(1);
|
|
2124
2209
|
}
|
|
2210
|
+
if (failedCount > 0) {
|
|
2211
|
+
// The summary names every failed document, and can be larger than a
|
|
2212
|
+
// pipe's buffer: it has to reach the pipe before the process ends.
|
|
2213
|
+
await flushOutput();
|
|
2214
|
+
process.exit(1);
|
|
2215
|
+
}
|
|
2125
2216
|
});
|
|
2126
2217
|
// Revoke group permission
|
|
2127
2218
|
groupPermissions
|
|
@@ -2253,6 +2344,12 @@ function chunkFileOf(chunk) {
|
|
|
2253
2344
|
async function exportSingleDocument(client, appId, documentId, outputDir, includeBlobs, jsonOutput) {
|
|
2254
2345
|
const docDir = path.join(outputDir, "documents", documentId);
|
|
2255
2346
|
fs.mkdirSync(docDir, { recursive: true });
|
|
2347
|
+
// `metadata.json` is what marks the directory a finished export — import
|
|
2348
|
+
// skips one without it — and it is written last. A rerun into an earlier
|
|
2349
|
+
// export's directory drops it before overwriting anything, so a run that
|
|
2350
|
+
// fails part-way leaves no new state beside the old aliases and grants
|
|
2351
|
+
// looking complete (#4149).
|
|
2352
|
+
fs.rmSync(path.join(docDir, "metadata.json"), { force: true });
|
|
2256
2353
|
// 1. Fetch document metadata
|
|
2257
2354
|
const docMeta = await client.getDocument(appId, documentId);
|
|
2258
2355
|
// 2. Export the document's state. A legacy document is one blob; a large
|
|
@@ -2268,35 +2365,32 @@ async function exportSingleDocument(client, appId, documentId, outputDir, includ
|
|
|
2268
2365
|
fs.writeFileSync(path.join(docDir, "document.yjs"), Buffer.from(stateResult.state, "base64"));
|
|
2269
2366
|
stateBytes = stateResult.byteLength;
|
|
2270
2367
|
}
|
|
2271
|
-
// 3.
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
//
|
|
2280
|
-
//
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2368
|
+
// 3. Read the permissions, pending invitations (deferred grants), aliases
|
|
2369
|
+
// and blob list — all before `permissions.json` or `metadata.json` is
|
|
2370
|
+
// written. Only a documented "none" answer reads as empty (a 404, and an
|
|
2371
|
+
// older server's 403/404 on pending invitations); any other failure fails
|
|
2372
|
+
// the document, because an export missing them reports success and the
|
|
2373
|
+
// import restores less than the document had (#4149).
|
|
2374
|
+
//
|
|
2375
|
+
// A root document cannot be shared, and the server answers both sharing
|
|
2376
|
+
// reads on one with a 403 (`validateNonRootDocument`). It has no grants
|
|
2377
|
+
// to export, so those reads are skipped rather than failing the document.
|
|
2378
|
+
const isRoot = isRootDocumentExport(docMeta?.tags);
|
|
2379
|
+
const permResult = isRoot
|
|
2380
|
+
? null
|
|
2381
|
+
: await readExportLookup(documentId, "permissions", () => client.listDocumentPermissions(appId, documentId));
|
|
2382
|
+
const pendingResult = isRoot
|
|
2383
|
+
? null
|
|
2384
|
+
: await readExportLookup(documentId, "pending invitations", () => client.listDocumentPendingInvitations(appId, documentId), { absentStatuses: PENDING_INVITATIONS_ABSENT_STATUSES });
|
|
2385
|
+
const aliasResult = await readExportLookup(documentId, "aliases", () => client.listDocumentAliases(appId, documentId));
|
|
2386
|
+
const blobs = includeBlobs
|
|
2387
|
+
? await readExportBlobList(client, appId, documentId)
|
|
2388
|
+
: [];
|
|
2389
|
+
// 4. Write permissions
|
|
2288
2390
|
const permissionsExport = buildPermissionsExport(permResult, pendingResult);
|
|
2289
2391
|
fs.writeFileSync(path.join(docDir, "permissions.json"), JSON.stringify(permissionsExport, null, 2));
|
|
2290
|
-
// 5.
|
|
2291
|
-
|
|
2292
|
-
try {
|
|
2293
|
-
const aliasResult = await client.listDocumentAliases(appId, documentId);
|
|
2294
|
-
aliases = (Array.isArray(aliasResult) ? aliasResult : aliasResult?.aliases || [])
|
|
2295
|
-
.filter((a) => (a.scope || a.aliasScope) === "user");
|
|
2296
|
-
}
|
|
2297
|
-
catch {
|
|
2298
|
-
// May not have aliases
|
|
2299
|
-
}
|
|
2392
|
+
// 5. Aliases (user-scoped only)
|
|
2393
|
+
const aliases = (Array.isArray(aliasResult) ? aliasResult : aliasResult?.aliases || []).filter((a) => (a.scope || a.aliasScope) === "user");
|
|
2300
2394
|
// 6. Write metadata. A large document says so here as well as in
|
|
2301
2395
|
// `chain.json`: the format decides how the artifact has to be installed,
|
|
2302
2396
|
// and an import that reads only `metadata.json` must not mistake one for
|
|
@@ -2316,14 +2410,6 @@ async function exportSingleDocument(client, appId, documentId, outputDir, includ
|
|
|
2316
2410
|
fs.writeFileSync(path.join(docDir, "metadata.json"), JSON.stringify(metadata, null, 2));
|
|
2317
2411
|
// 7. Blobs
|
|
2318
2412
|
if (includeBlobs) {
|
|
2319
|
-
let blobs = [];
|
|
2320
|
-
try {
|
|
2321
|
-
const blobResult = await client.listDocumentBlobs(appId, documentId);
|
|
2322
|
-
blobs = Array.isArray(blobResult) ? blobResult : blobResult?.items || blobResult?.blobs || [];
|
|
2323
|
-
}
|
|
2324
|
-
catch {
|
|
2325
|
-
// May not have blobs
|
|
2326
|
-
}
|
|
2327
2413
|
if (blobs.length > 0) {
|
|
2328
2414
|
const blobsDir = path.join(docDir, "blobs");
|
|
2329
2415
|
fs.mkdirSync(blobsDir, { recursive: true });
|
|
@@ -2356,6 +2442,40 @@ async function exportSingleDocument(client, appId, documentId, outputDir, includ
|
|
|
2356
2442
|
success(`Exported document ${documentId} (${metadata.title || "untitled"}) to ${docDir}`);
|
|
2357
2443
|
}
|
|
2358
2444
|
}
|
|
2445
|
+
/**
|
|
2446
|
+
* Every uploaded blob of a document, read page by page (#4149).
|
|
2447
|
+
*
|
|
2448
|
+
* The list answers one page at a time (`{ items, hasMore, nextCursor }`), so
|
|
2449
|
+
* a document with more blobs than a page holds would otherwise export only
|
|
2450
|
+
* the first page and report success. Each page is its own lookup: a 404 on
|
|
2451
|
+
* the first means the document has no blobs, but a later page that fails —
|
|
2452
|
+
* a 404 included — fails the document, because the list is already known to
|
|
2453
|
+
* be longer. A page that says more follow without a new cursor to read them
|
|
2454
|
+
* by fails the document too: stopping there exports part of the list, and
|
|
2455
|
+
* rereading the same cursor never ends.
|
|
2456
|
+
*/
|
|
2457
|
+
async function readExportBlobList(client, appId, documentId) {
|
|
2458
|
+
const blobs = [];
|
|
2459
|
+
const seen = new Set();
|
|
2460
|
+
let cursor;
|
|
2461
|
+
do {
|
|
2462
|
+
const pageCursor = cursor;
|
|
2463
|
+
const page = await readExportLookup(documentId, "blob list", () => client.listDocumentBlobs(appId, documentId, pageCursor ? { cursor: pageCursor } : undefined), pageCursor ? { absentStatuses: [] } : {});
|
|
2464
|
+
if (Array.isArray(page))
|
|
2465
|
+
return [...blobs, ...page];
|
|
2466
|
+
blobs.push(...(page?.items || page?.blobs || []));
|
|
2467
|
+
if (!page?.hasMore)
|
|
2468
|
+
break;
|
|
2469
|
+
cursor = page?.nextCursor || undefined;
|
|
2470
|
+
if (!cursor || seen.has(cursor)) {
|
|
2471
|
+
throw new Error(`Document ${documentId}: reading its blob list failed — a page said ` +
|
|
2472
|
+
`more follow but gave no new cursor to read them by. Its export ` +
|
|
2473
|
+
`would be incomplete, so it was not exported.`);
|
|
2474
|
+
}
|
|
2475
|
+
seen.add(cursor);
|
|
2476
|
+
} while (cursor);
|
|
2477
|
+
return blobs;
|
|
2478
|
+
}
|
|
2359
2479
|
/**
|
|
2360
2480
|
* Whether `docDir` holds a large document's export (#2816, behavior 20).
|
|
2361
2481
|
*
|
|
@@ -2485,6 +2605,24 @@ function verifyChainArtifact(docDir, documentId, chain) {
|
|
|
2485
2605
|
}
|
|
2486
2606
|
require(path.join(docDir, "current.yjs"), "its open epoch's overlay");
|
|
2487
2607
|
}
|
|
2608
|
+
/**
|
|
2609
|
+
* One idempotent write of an import, retried through a transient failure
|
|
2610
|
+
* (#4148): a blob PUT, a state merge, an artifact upload or an install read.
|
|
2611
|
+
* Each retry is said, so a slow run is not mistaken for a stuck one.
|
|
2612
|
+
*/
|
|
2613
|
+
function retryingImportStep(what, attempt) {
|
|
2614
|
+
return withTransientRetry(attempt, (failure, delayMs) => warn(`${what} failed (${failure?.message || "no reason given"}); ` +
|
|
2615
|
+
`retrying in ${delayMs / 1000}s`));
|
|
2616
|
+
}
|
|
2617
|
+
/** Upload one chain artifact; the server stores it under a fixed key. */
|
|
2618
|
+
function uploadChainArtifact(client, appId, documentId, query, body) {
|
|
2619
|
+
const what = query.kind === "chunk"
|
|
2620
|
+
? `chunk ${query.index} of ${query.model}`
|
|
2621
|
+
: query.kind === "overlay"
|
|
2622
|
+
? `the overlay of epoch ${query.epoch}`
|
|
2623
|
+
: "the base snapshot's manifest";
|
|
2624
|
+
return retryingImportStep(`Uploading ${what} of ${documentId}`, () => client.uploadDocumentChainArtifact(appId, documentId, query, body));
|
|
2625
|
+
}
|
|
2488
2626
|
/** How long an install is waited on before the CLI gives up on it. */
|
|
2489
2627
|
const CHAIN_INSTALL_TIMEOUT_MS = 60 * 60 * 1000;
|
|
2490
2628
|
/** How often the install's progress is asked for. */
|
|
@@ -2503,12 +2641,17 @@ const CHAIN_INSTALL_POLL_MS = 1000;
|
|
|
2503
2641
|
*/
|
|
2504
2642
|
async function importLargeDocumentChain(client, appId, docDir, documentId) {
|
|
2505
2643
|
const chain = readChainArtifact(docDir, documentId);
|
|
2644
|
+
await uploadAndInstallChain(client, appId, docDir, documentId, chain);
|
|
2645
|
+
await awaitChainInstall(client, appId, documentId);
|
|
2646
|
+
}
|
|
2647
|
+
/** Upload every artifact of the chain, then start installing it. */
|
|
2648
|
+
async function uploadAndInstallChain(client, appId, docDir, documentId, chain) {
|
|
2506
2649
|
// The base's manifest and chunks. The manifest names the chunks, so nothing
|
|
2507
2650
|
// here has to walk the directory and guess what belongs to the build.
|
|
2508
2651
|
if (chain.base) {
|
|
2509
2652
|
const manifestPath = path.join(docDir, "snapshot", "manifest.json");
|
|
2510
2653
|
const manifestBytes = fs.readFileSync(manifestPath);
|
|
2511
|
-
await client
|
|
2654
|
+
await uploadChainArtifact(client, appId, documentId, {
|
|
2512
2655
|
kind: "manifest",
|
|
2513
2656
|
epoch: chain.base.epoch,
|
|
2514
2657
|
buildId: chain.base.buildId,
|
|
@@ -2517,7 +2660,7 @@ async function importLargeDocumentChain(client, appId, docDir, documentId) {
|
|
|
2517
2660
|
for (const chunk of manifest.chunks ?? []) {
|
|
2518
2661
|
// Addressed by the entry's `path` (#3432); the server installs by it too.
|
|
2519
2662
|
const { model, index } = chunkFileOf(chunk);
|
|
2520
|
-
await client
|
|
2663
|
+
await uploadChainArtifact(client, appId, documentId, {
|
|
2521
2664
|
kind: "chunk",
|
|
2522
2665
|
epoch: chain.base.epoch,
|
|
2523
2666
|
buildId: chain.base.buildId,
|
|
@@ -2530,9 +2673,9 @@ async function importLargeDocumentChain(client, appId, docDir, documentId) {
|
|
|
2530
2673
|
// archived exactly like a sealed one, so the imported document's chain is
|
|
2531
2674
|
// the chain that was exported.
|
|
2532
2675
|
for (const overlay of chain.overlays) {
|
|
2533
|
-
await client
|
|
2676
|
+
await uploadChainArtifact(client, appId, documentId, { kind: "overlay", epoch: overlay.epoch }, fs.readFileSync(path.join(docDir, "epochs", `${overlay.epoch}.yjs`)));
|
|
2534
2677
|
}
|
|
2535
|
-
await client
|
|
2678
|
+
await uploadChainArtifact(client, appId, documentId, { kind: "overlay", epoch: chain.epoch }, fs.readFileSync(path.join(docDir, "current.yjs")));
|
|
2536
2679
|
await client.installDocumentChain(appId, documentId, {
|
|
2537
2680
|
base: chain.base
|
|
2538
2681
|
? { epoch: chain.base.epoch, buildId: chain.base.buildId }
|
|
@@ -2540,19 +2683,25 @@ async function importLargeDocumentChain(client, appId, docDir, documentId) {
|
|
|
2540
2683
|
overlays: chain.overlays.map((overlay) => overlay.epoch),
|
|
2541
2684
|
currentEpoch: chain.epoch,
|
|
2542
2685
|
});
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2686
|
+
}
|
|
2687
|
+
/**
|
|
2688
|
+
* Wait for the document's chain install to finish.
|
|
2689
|
+
*
|
|
2690
|
+
* The install runs on the document's alarm, a bounded tick at a time, so it
|
|
2691
|
+
* is waited on rather than awaited: reporting "imported" while the records
|
|
2692
|
+
* are still arriving is the failure this whole path exists to avoid.
|
|
2693
|
+
*/
|
|
2694
|
+
async function awaitChainInstall(client, appId, documentId) {
|
|
2546
2695
|
const deadline = Date.now() + CHAIN_INSTALL_TIMEOUT_MS;
|
|
2547
2696
|
for (;;) {
|
|
2548
|
-
const status = await client.getDocumentChainInstall(appId, documentId);
|
|
2697
|
+
const status = await retryingImportStep(`Reading the chain install of ${documentId}`, () => client.getDocumentChainInstall(appId, documentId));
|
|
2549
2698
|
const install = status?.install;
|
|
2550
2699
|
if (install?.state === "complete")
|
|
2551
2700
|
return;
|
|
2552
2701
|
if (install?.state === "failed") {
|
|
2553
2702
|
throw new Error(`Document ${documentId}: installing its chain failed — ` +
|
|
2554
|
-
`${install.error || "no reason given"}. The document is incomplete ` +
|
|
2555
|
-
`
|
|
2703
|
+
`${install.error || "no reason given"}. The document is incomplete; ` +
|
|
2704
|
+
`re-run the import to install it again.`);
|
|
2556
2705
|
}
|
|
2557
2706
|
if (Date.now() >= deadline) {
|
|
2558
2707
|
throw new Error(`Document ${documentId}: its chain install has not finished after ` +
|
|
@@ -2568,19 +2717,6 @@ const ROOT_DOCUMENT_TAG = "__ROOT_TAG__";
|
|
|
2568
2717
|
function isRootDocumentExport(tags) {
|
|
2569
2718
|
return Array.isArray(tags) && tags.includes(ROOT_DOCUMENT_TAG);
|
|
2570
2719
|
}
|
|
2571
|
-
/**
|
|
2572
|
-
* Why a large-document root export is refused (#3135, contract 7).
|
|
2573
|
-
*
|
|
2574
|
-
* A format-2 export is a chain installed wholesale into a freshly created
|
|
2575
|
-
* format-2 document — a replace, not a merge, and not something that can be
|
|
2576
|
-
* done to a root document, which the server only ever mints as a legacy
|
|
2577
|
-
* document. Refusing says so rather than installing a chain somewhere it does
|
|
2578
|
-
* not belong.
|
|
2579
|
-
*/
|
|
2580
|
-
const ROOT_LARGE_DOCUMENT_REFUSAL = "it is a root document exported as a large document (documentFormat 2). A " +
|
|
2581
|
-
"root document is minted as a legacy document and a chain is installed " +
|
|
2582
|
-
"wholesale into a new format-2 document, so the export cannot be applied " +
|
|
2583
|
-
"to the target user's root. Nothing was imported for it.";
|
|
2584
2720
|
/** Read an export directory's `metadata.json`, or null when it has none. */
|
|
2585
2721
|
function readExportMetadata(docDir) {
|
|
2586
2722
|
const metadataPath = path.join(docDir, "metadata.json");
|
|
@@ -2610,6 +2746,24 @@ function readExportMetadata(docDir) {
|
|
|
2610
2746
|
*/
|
|
2611
2747
|
async function planRootImports(client, appId, docDirs, explicitOwner, explicitOwnerUserId, overwrite) {
|
|
2612
2748
|
const plans = new Map();
|
|
2749
|
+
// The app's `rootDocumentFormat`, read once and only when some target user
|
|
2750
|
+
// has no root yet (#4195). A failed read fails the run: a guessed format is
|
|
2751
|
+
// a confident create into the wrong kind of root.
|
|
2752
|
+
let appSetting;
|
|
2753
|
+
const readAppSetting = async (exportedDocumentId) => {
|
|
2754
|
+
if (!appSetting) {
|
|
2755
|
+
try {
|
|
2756
|
+
const app = await client.getApp(appId);
|
|
2757
|
+
appSetting = { value: app?.rootDocumentFormat ?? null };
|
|
2758
|
+
}
|
|
2759
|
+
catch (error) {
|
|
2760
|
+
throw new Error(`Document ${exportedDocumentId}: reading app ${appId}'s ` +
|
|
2761
|
+
`rootDocumentFormat failed — ${error?.message || "no reason given"}. ` +
|
|
2762
|
+
`Nothing was imported.`);
|
|
2763
|
+
}
|
|
2764
|
+
}
|
|
2765
|
+
return appSetting.value;
|
|
2766
|
+
};
|
|
2613
2767
|
for (const docDir of docDirs) {
|
|
2614
2768
|
const metadata = readExportMetadata(docDir);
|
|
2615
2769
|
if (!metadata || !isRootDocumentExport(metadata.tags))
|
|
@@ -2621,16 +2775,13 @@ async function planRootImports(client, appId, docDirs, explicitOwner, explicitOw
|
|
|
2621
2775
|
exportedDocumentId,
|
|
2622
2776
|
owner,
|
|
2623
2777
|
action: "refused",
|
|
2778
|
+
exportFormat: rootDocumentFormatOf(metadata.documentFormat),
|
|
2624
2779
|
};
|
|
2625
2780
|
const refuse = (reason) => {
|
|
2626
2781
|
plan.action = "refused";
|
|
2627
2782
|
plan.reason = reason;
|
|
2628
2783
|
plans.set(docDir, plan);
|
|
2629
2784
|
};
|
|
2630
|
-
if (Number(metadata.documentFormat) === 2) {
|
|
2631
|
-
refuse(ROOT_LARGE_DOCUMENT_REFUSAL);
|
|
2632
|
-
continue;
|
|
2633
|
-
}
|
|
2634
2785
|
if (!owner) {
|
|
2635
2786
|
refuse(`it is a root document export that records no owner, so there is no ` +
|
|
2636
2787
|
`user in ${appId} to restore it into. Pass --owner <userId-or-email>.`);
|
|
@@ -2678,13 +2829,33 @@ async function planRootImports(client, appId, docDirs, explicitOwner, explicitOw
|
|
|
2678
2829
|
refuse(unresolvableRootOwner(appId, owner));
|
|
2679
2830
|
continue;
|
|
2680
2831
|
}
|
|
2832
|
+
// The export's format against the root it would land in (#4195): the
|
|
2833
|
+
// existing root's, or the one the app's setting says the mint produces.
|
|
2681
2834
|
const rootDocId = assignment?.rootDocId || null;
|
|
2682
|
-
if (
|
|
2683
|
-
plan.
|
|
2835
|
+
if (rootDocId)
|
|
2836
|
+
plan.targetRootDocId = rootDocId;
|
|
2837
|
+
const decision = decideRootImport({
|
|
2838
|
+
exportFormat: plan.exportFormat,
|
|
2839
|
+
existingRootFormat: rootDocId
|
|
2840
|
+
? rootDocumentFormatOf(assignment.documentFormat)
|
|
2841
|
+
: null,
|
|
2842
|
+
appSetting: rootDocId ? null : await readAppSetting(exportedDocumentId),
|
|
2843
|
+
overwrite,
|
|
2844
|
+
owner,
|
|
2845
|
+
...(rootDocId ? { targetRootDocId: rootDocId } : {}),
|
|
2846
|
+
});
|
|
2847
|
+
plan.targetFormat = decision.targetFormat;
|
|
2848
|
+
if (decision.outcome === "refuse") {
|
|
2849
|
+
refuse(decision.reason);
|
|
2850
|
+
continue;
|
|
2851
|
+
}
|
|
2852
|
+
if (decision.outcome === "skip") {
|
|
2853
|
+
plan.action = "skip";
|
|
2854
|
+
plan.reason = decision.reason;
|
|
2684
2855
|
}
|
|
2685
2856
|
else {
|
|
2686
|
-
plan.
|
|
2687
|
-
plan.
|
|
2857
|
+
plan.action = rootDocId ? "apply" : "create";
|
|
2858
|
+
plan.mode = decision.outcome;
|
|
2688
2859
|
}
|
|
2689
2860
|
plans.set(docDir, plan);
|
|
2690
2861
|
}
|
|
@@ -2727,7 +2898,7 @@ function assertDistinctRootTargets(plans) {
|
|
|
2727
2898
|
}
|
|
2728
2899
|
}
|
|
2729
2900
|
/**
|
|
2730
|
-
*
|
|
2901
|
+
* The target app's copy of this document, or null when it holds none.
|
|
2731
2902
|
*
|
|
2732
2903
|
* Only an explicit 404 answers "no" (#3096). An expired login (401), a lost
|
|
2733
2904
|
* permission (403), a server fault or a transport failure says nothing about
|
|
@@ -2736,19 +2907,18 @@ function assertDistinctRootTargets(plans) {
|
|
|
2736
2907
|
* silently omits the overwrite the operator asked for. Those propagate, and
|
|
2737
2908
|
* `documents import` fails the run.
|
|
2738
2909
|
*/
|
|
2739
|
-
async function
|
|
2910
|
+
async function existingDocument(client, appId, documentId) {
|
|
2740
2911
|
try {
|
|
2741
|
-
await client.getDocument(appId, documentId);
|
|
2742
|
-
return true;
|
|
2912
|
+
return (await client.getDocument(appId, documentId)) ?? {};
|
|
2743
2913
|
}
|
|
2744
2914
|
catch (error) {
|
|
2745
2915
|
if (error?.statusCode === 404)
|
|
2746
|
-
return
|
|
2916
|
+
return null;
|
|
2747
2917
|
throw new Error(`Document ${documentId}: checking whether it already exists failed — ` +
|
|
2748
2918
|
`${error?.message || "no reason given"}. Nothing was imported for it.`);
|
|
2749
2919
|
}
|
|
2750
2920
|
}
|
|
2751
|
-
async function importSingleDocument(client, appId, docDir, overwrite, aliasMode, dryRun, summary, ownerUserId, rootPlan) {
|
|
2921
|
+
async function importSingleDocument(client, appId, docDir, overwrite, aliasMode, dryRun, summary, progress, ownerUserId, rootPlan) {
|
|
2752
2922
|
const metadataPath = path.join(docDir, "metadata.json");
|
|
2753
2923
|
if (!fs.existsSync(metadataPath)) {
|
|
2754
2924
|
warn(`Skipping ${docDir}: no metadata.json found`);
|
|
@@ -2761,12 +2931,12 @@ async function importSingleDocument(client, appId, docDir, overwrite, aliasMode,
|
|
|
2761
2931
|
// the SOURCE user's root document. It goes to the target user's own root
|
|
2762
2932
|
// instead, per the plan made before this run wrote anything (#3135).
|
|
2763
2933
|
//
|
|
2764
|
-
// Decided BEFORE the chain artifact below is read
|
|
2765
|
-
//
|
|
2766
|
-
//
|
|
2767
|
-
// run
|
|
2934
|
+
// Decided BEFORE the chain artifact below is read: a root export's chain is
|
|
2935
|
+
// installed into the target user's root, never under its exported id, and
|
|
2936
|
+
// only when the plan says so (#4195) — reading it here would fail the whole
|
|
2937
|
+
// run over an artifact a refused or skipped plan never touches.
|
|
2768
2938
|
if (isRootDocumentExport(metadata.tags)) {
|
|
2769
|
-
await importRootDocument(client, appId, docDir, metadata, rootPlan, overwrite, aliasMode, dryRun, summary);
|
|
2939
|
+
await importRootDocument(client, appId, docDir, metadata, rootPlan, overwrite, aliasMode, dryRun, summary, progress);
|
|
2770
2940
|
return;
|
|
2771
2941
|
}
|
|
2772
2942
|
// A large document's data is the chain, not `document.yjs` (#2816), and it
|
|
@@ -2777,9 +2947,78 @@ async function importSingleDocument(client, appId, docDir, overwrite, aliasMode,
|
|
|
2777
2947
|
// `--dry-run` decides from the same facts the real run does (#3096), so a
|
|
2778
2948
|
// preview never promises a skip that the run turns into a merge: it looks
|
|
2779
2949
|
// the document up as well, and only the writes below are withheld.
|
|
2780
|
-
const
|
|
2950
|
+
const existing = await existingDocument(client, appId, documentId);
|
|
2951
|
+
const exists = existing !== null;
|
|
2952
|
+
progress.exists = exists;
|
|
2781
2953
|
const wouldSkip = dryRun ? "[dry-run] Would skip" : "Skipping";
|
|
2782
2954
|
if (exists && !overwrite) {
|
|
2955
|
+
// A same-id document of the other format is not one this import created:
|
|
2956
|
+
// the format is decided at creation and never migrated. It is skipped
|
|
2957
|
+
// before any read of what it lacks, which its format's endpoints refuse.
|
|
2958
|
+
if ((Number(existing?.documentFormat) === 2) !== isLargeDocument) {
|
|
2959
|
+
warn(`${wouldSkip} ${documentId}: already exists as a ` +
|
|
2960
|
+
`${isLargeDocument ? "legacy" : "large"} document, and the export ` +
|
|
2961
|
+
`is a ${isLargeDocument ? "large" : "legacy"} one`);
|
|
2962
|
+
summary.skipped++;
|
|
2963
|
+
return;
|
|
2964
|
+
}
|
|
2965
|
+
// A document an earlier run created and did not finish is completed, not
|
|
2966
|
+
// skipped (#4148): what it lacks of the export is read from the server,
|
|
2967
|
+
// and only that is written. A document that lacks nothing, or that holds
|
|
2968
|
+
// content of its own, is skipped, so a re-run never merges into or
|
|
2969
|
+
// re-uploads to one already imported, nor writes to one it did not start.
|
|
2970
|
+
// With an owner named, the export's user aliases are among what it can
|
|
2971
|
+
// lack.
|
|
2972
|
+
const gaps = await findImportGaps(client, appId, docDir, documentId, isLargeDocument, ownerUserId
|
|
2973
|
+
? { userId: ownerUserId, aliasMode, exported: metadata.aliases }
|
|
2974
|
+
: undefined);
|
|
2975
|
+
// Never installed and not empty: a document of its own, which the server
|
|
2976
|
+
// would refuse to install into.
|
|
2977
|
+
const skipHoldingRecords = (held = "holds records of its own") => {
|
|
2978
|
+
warn(`${wouldSkip} ${documentId}: already exists and ${held}, so its ` +
|
|
2979
|
+
`large-document export cannot be installed into it`);
|
|
2980
|
+
summary.skipped++;
|
|
2981
|
+
};
|
|
2982
|
+
if (gaps === "holds-records") {
|
|
2983
|
+
skipHoldingRecords();
|
|
2984
|
+
return;
|
|
2985
|
+
}
|
|
2986
|
+
if (gaps === "rotated-epoch") {
|
|
2987
|
+
skipHoldingRecords("has rotated an epoch of its own");
|
|
2988
|
+
return;
|
|
2989
|
+
}
|
|
2990
|
+
if (gaps) {
|
|
2991
|
+
if (dryRun) {
|
|
2992
|
+
info(`[dry-run] Would complete document ${documentId} ` +
|
|
2993
|
+
`(${metadata.title || "untitled"}): it is missing ${describeImportGaps(gaps)}`);
|
|
2994
|
+
summary.completed++;
|
|
2995
|
+
return;
|
|
2996
|
+
}
|
|
2997
|
+
if (gaps.state)
|
|
2998
|
+
await importExportedState(client, appId, docDir, documentId);
|
|
2999
|
+
if (gaps.chain === "install") {
|
|
3000
|
+
try {
|
|
3001
|
+
await uploadAndInstallChain(client, appId, docDir, documentId, readChainArtifact(docDir, documentId));
|
|
3002
|
+
}
|
|
3003
|
+
catch (refusal) {
|
|
3004
|
+
// The server's own check, at the first artifact and before it
|
|
3005
|
+
// stores anything: it decides when the document changed after the
|
|
3006
|
+
// reads above.
|
|
3007
|
+
if (refusal?.code !== "DOCUMENT_NOT_IMPORTABLE")
|
|
3008
|
+
throw refusal;
|
|
3009
|
+
skipHoldingRecords();
|
|
3010
|
+
return;
|
|
3011
|
+
}
|
|
3012
|
+
}
|
|
3013
|
+
if (gaps.chain)
|
|
3014
|
+
await awaitChainInstall(client, appId, documentId);
|
|
3015
|
+
await uploadExportedBlobs(client, appId, documentId, docDir, gaps.blobs, summary);
|
|
3016
|
+
await restoreExportedAliases(client, appId, documentId, ownerUserId ? { ...metadata, aliases: gaps.aliases } : metadata, ownerUserId, aliasMode, summary);
|
|
3017
|
+
summary.completed++;
|
|
3018
|
+
info(`Completed document ${documentId} (${metadata.title || "untitled"}): ` +
|
|
3019
|
+
`imported ${describeImportGaps(gaps)}`);
|
|
3020
|
+
return;
|
|
3021
|
+
}
|
|
2783
3022
|
// `--overwrite` merges a legacy document's state — but a large document is
|
|
2784
3023
|
// INSTALLED, and the server refuses to install a chain into a document
|
|
2785
3024
|
// that already holds records (`assertDocumentIsImportable`), so the remedy
|
|
@@ -2817,6 +3056,7 @@ async function importSingleDocument(client, appId, docDir, overwrite, aliasMode,
|
|
|
2817
3056
|
...(isLargeDocument ? { documentFormat: 2 } : {}),
|
|
2818
3057
|
...(ownerUserId ? { createdBy: ownerUserId } : {}),
|
|
2819
3058
|
});
|
|
3059
|
+
progress.exists = true;
|
|
2820
3060
|
summary.created++;
|
|
2821
3061
|
}
|
|
2822
3062
|
else {
|
|
@@ -2828,57 +3068,254 @@ async function importSingleDocument(client, appId, docDir, overwrite, aliasMode,
|
|
|
2828
3068
|
await importLargeDocumentChain(client, appId, docDir, documentId);
|
|
2829
3069
|
}
|
|
2830
3070
|
else {
|
|
2831
|
-
|
|
2832
|
-
const yjsPath = path.join(docDir, "document.yjs");
|
|
2833
|
-
if (fs.existsSync(yjsPath)) {
|
|
2834
|
-
const stateBuffer = fs.readFileSync(yjsPath);
|
|
2835
|
-
const stateBase64 = stateBuffer.toString("base64");
|
|
2836
|
-
await client.importDocumentState(appId, documentId, stateBase64);
|
|
2837
|
-
}
|
|
3071
|
+
await importExportedState(client, appId, docDir, documentId);
|
|
2838
3072
|
}
|
|
2839
|
-
|
|
3073
|
+
await uploadExportedBlobs(client, appId, documentId, docDir, exportedBlobsOf(docDir), summary);
|
|
3074
|
+
// Permissions are exported for reference but not restored during import.
|
|
3075
|
+
// The importing admin is the new owner; sharing is managed in the target app.
|
|
3076
|
+
await restoreExportedAliases(client, appId, documentId, metadata, ownerUserId, aliasMode, summary);
|
|
3077
|
+
info(`Imported document ${documentId} (${metadata.title || "untitled"})`);
|
|
3078
|
+
}
|
|
3079
|
+
/** Merge the exported Yjs state into the document, when the export has one. */
|
|
3080
|
+
async function importExportedState(client, appId, docDir, documentId) {
|
|
3081
|
+
const yjsPath = path.join(docDir, "document.yjs");
|
|
3082
|
+
if (!fs.existsSync(yjsPath))
|
|
3083
|
+
return;
|
|
3084
|
+
const stateBase64 = fs.readFileSync(yjsPath).toString("base64");
|
|
3085
|
+
await retryingImportStep(`Importing the state of ${documentId}`, () => client.importDocumentState(appId, documentId, stateBase64));
|
|
3086
|
+
}
|
|
3087
|
+
/** The blobs the export lists whose bytes are in it. */
|
|
3088
|
+
function exportedBlobsOf(docDir) {
|
|
2840
3089
|
const blobIndexPath = path.join(docDir, "blobs", "index.json");
|
|
2841
|
-
if (fs.existsSync(blobIndexPath))
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
3090
|
+
if (!fs.existsSync(blobIndexPath))
|
|
3091
|
+
return [];
|
|
3092
|
+
const blobIndex = JSON.parse(fs.readFileSync(blobIndexPath, "utf-8"));
|
|
3093
|
+
return (Array.isArray(blobIndex) ? blobIndex : []).filter((blob) => fs.existsSync(path.join(docDir, "blobs", `${blob.blobId}.bin`)));
|
|
3094
|
+
}
|
|
3095
|
+
/** Upload exported blobs under their own ids. */
|
|
3096
|
+
async function uploadExportedBlobs(client, appId, documentId, docDir, blobs, summary) {
|
|
3097
|
+
for (const blob of blobs) {
|
|
3098
|
+
const data = fs.readFileSync(path.join(docDir, "blobs", `${blob.blobId}.bin`));
|
|
3099
|
+
await retryingImportStep(`Uploading blob ${blob.blobId} of ${documentId}`, () => client.uploadBlob(appId, documentId, blob.blobId, data, {
|
|
3100
|
+
filename: blob.filename,
|
|
3101
|
+
contentType: blob.contentType,
|
|
3102
|
+
sha256: blob.sha256,
|
|
3103
|
+
}));
|
|
3104
|
+
summary.blobsUploaded++;
|
|
3105
|
+
}
|
|
3106
|
+
}
|
|
3107
|
+
/** Restore the export's user-scoped aliases, when an owner was named. */
|
|
3108
|
+
async function restoreExportedAliases(client, appId, documentId, metadata, ownerUserId, aliasMode, summary) {
|
|
3109
|
+
if (!Array.isArray(metadata.aliases))
|
|
3110
|
+
return;
|
|
3111
|
+
for (const alias of metadata.aliases) {
|
|
3112
|
+
if (alias.aliasScope !== "user")
|
|
3113
|
+
continue;
|
|
3114
|
+
if (!ownerUserId) {
|
|
3115
|
+
summary.aliasesSkipped++;
|
|
3116
|
+
continue;
|
|
3117
|
+
}
|
|
3118
|
+
try {
|
|
3119
|
+
await retryingImportStep(`Setting alias ${alias.aliasKey} of ${documentId}`, () => client.setDocumentAlias(appId, alias.aliasScope, alias.aliasKey, documentId, ownerUserId, aliasMode === "skip" // mustNotExist: true for skip mode, false for overwrite
|
|
3120
|
+
));
|
|
3121
|
+
summary.aliasesSet++;
|
|
3122
|
+
}
|
|
3123
|
+
catch (failure) {
|
|
3124
|
+
// An alias the user already holds is the answer `--aliases skip` asks
|
|
3125
|
+
// for, and it is left where it is. Any other failure leaves the
|
|
3126
|
+
// document without an alias its export holds: it is incomplete, and
|
|
3127
|
+
// reported as such (#4148).
|
|
3128
|
+
if (failure?.statusCode !== 409)
|
|
3129
|
+
throw failure;
|
|
3130
|
+
summary.aliasesSkipped++;
|
|
2854
3131
|
}
|
|
2855
3132
|
}
|
|
2856
|
-
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
2860
|
-
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
|
|
3133
|
+
}
|
|
3134
|
+
/** "its state and 2 blobs", for the line that reports a completion. */
|
|
3135
|
+
function describeImportGaps(gaps) {
|
|
3136
|
+
const parts = [];
|
|
3137
|
+
if (gaps.state)
|
|
3138
|
+
parts.push("its state");
|
|
3139
|
+
if (gaps.chain === "install")
|
|
3140
|
+
parts.push("its chain");
|
|
3141
|
+
if (gaps.chain === "await")
|
|
3142
|
+
parts.push("its chain (installing)");
|
|
3143
|
+
if (gaps.blobs.length > 0) {
|
|
3144
|
+
parts.push(`${gaps.blobs.length} blob${gaps.blobs.length === 1 ? "" : "s"}`);
|
|
3145
|
+
}
|
|
3146
|
+
if (gaps.aliases.length > 0) {
|
|
3147
|
+
parts.push(`${gaps.aliases.length} alias${gaps.aliases.length === 1 ? "" : "es"}`);
|
|
3148
|
+
}
|
|
3149
|
+
return parts.join(" and ");
|
|
3150
|
+
}
|
|
3151
|
+
/** A Yjs update that holds nothing: no items and no deletions. */
|
|
3152
|
+
function isEmptyYjsUpdate(update) {
|
|
3153
|
+
return update.length === 0 || (update.length === 2 && update[0] === 0 && update[1] === 0);
|
|
3154
|
+
}
|
|
3155
|
+
/** The export's Yjs state, or null when it has none or an empty one. */
|
|
3156
|
+
function exportedStateOf(docDir) {
|
|
3157
|
+
const yjsPath = path.join(docDir, "document.yjs");
|
|
3158
|
+
if (!fs.existsSync(yjsPath))
|
|
3159
|
+
return null;
|
|
3160
|
+
const state = fs.readFileSync(yjsPath);
|
|
3161
|
+
return isEmptyYjsUpdate(state) ? null : state;
|
|
3162
|
+
}
|
|
3163
|
+
/**
|
|
3164
|
+
* Whether `held` already contains all of `exported`: for each client the
|
|
3165
|
+
* export wrote as, the document holds at least as far as the export does, and
|
|
3166
|
+
* it holds every deletion the export does. A deletion advances no clock, so
|
|
3167
|
+
* the clocks alone take a document read before a deletion for one after it.
|
|
3168
|
+
*/
|
|
3169
|
+
function holdsYjsUpdate(held, exported) {
|
|
3170
|
+
const heldClocks = Y.decodeStateVector(Y.encodeStateVectorFromUpdate(held));
|
|
3171
|
+
const exportedClocks = Y.decodeStateVector(Y.encodeStateVectorFromUpdate(exported));
|
|
3172
|
+
for (const [clientId, clock] of exportedClocks) {
|
|
3173
|
+
if ((heldClocks.get(clientId) ?? 0) < clock)
|
|
3174
|
+
return false;
|
|
3175
|
+
}
|
|
3176
|
+
// Every item the export deletes is held, so applying it changes the held
|
|
3177
|
+
// snapshot only if it deletes one the document has not.
|
|
3178
|
+
const doc = new Y.Doc();
|
|
3179
|
+
Y.applyUpdate(doc, held);
|
|
3180
|
+
const before = Y.snapshot(doc);
|
|
3181
|
+
Y.applyUpdate(doc, exported);
|
|
3182
|
+
return Y.equalSnapshots(before, Y.snapshot(doc));
|
|
3183
|
+
}
|
|
3184
|
+
/**
|
|
3185
|
+
* Read what an existing document lacks of its export (#4148): the gaps, null
|
|
3186
|
+
* when there is nothing a re-run may write, or `"holds-records"` /
|
|
3187
|
+
* `"rotated-epoch"` for a large document that was never imported into and
|
|
3188
|
+
* holds records or has rotated an epoch of its own.
|
|
3189
|
+
*
|
|
3190
|
+
* A legacy document is only completed when it is the one an earlier run
|
|
3191
|
+
* started: it holds no state at all — what `createDocument` leaves, and what a
|
|
3192
|
+
* failed `import/state` leaves, since the server stores the merge before it
|
|
3193
|
+
* applies it — or it already holds the exported state, and then only its
|
|
3194
|
+
* blobs can be missing. A document holding state of its own is not written
|
|
3195
|
+
* to at all, neither merged into nor given blobs or aliases: that is
|
|
3196
|
+
* `--overwrite`'s decision, not a re-run's. A large document's chain is
|
|
3197
|
+
* missing unless its install completed; the server takes a new install over a
|
|
3198
|
+
* failed one, but never one into a document with records and no install
|
|
3199
|
+
* behind them. Blobs are compared by id against every page the document
|
|
3200
|
+
* lists. When `aliasOwner` is given, the export's user aliases are read as
|
|
3201
|
+
* well: one the owner holds for another document counts as missing only when
|
|
3202
|
+
* the alias mode would move it.
|
|
3203
|
+
*/
|
|
3204
|
+
async function findImportGaps(client, appId, docDir, documentId, isLargeDocument, aliasOwner) {
|
|
3205
|
+
let state = false;
|
|
3206
|
+
let chain = null;
|
|
3207
|
+
const exported = exportedBlobsOf(docDir);
|
|
3208
|
+
if (isLargeDocument) {
|
|
3209
|
+
const status = await retryingImportStep(`Reading the chain install of ${documentId}`, () => client.getDocumentChainInstall(appId, documentId));
|
|
3210
|
+
const installState = status?.install?.state;
|
|
3211
|
+
if (installState === "installing")
|
|
3212
|
+
chain = "await";
|
|
3213
|
+
else if (installState !== "complete")
|
|
3214
|
+
chain = "install";
|
|
3215
|
+
// Read, not discovered by the first artifact upload, so `--dry-run`
|
|
3216
|
+
// reaches the same decision the run does.
|
|
3217
|
+
// `tableModels` is every model the document's records table holds.
|
|
3218
|
+
if (chain === "install" && !status?.install) {
|
|
3219
|
+
const schema = await retryingImportStep(`Reading the records of ${documentId}`, () => client.getDocumentSchema(appId, documentId));
|
|
3220
|
+
if (Array.isArray(schema?.tableModels) && schema.tableModels.length > 0) {
|
|
3221
|
+
return "holds-records";
|
|
3222
|
+
}
|
|
3223
|
+
// The server refuses one past its first epoch as well. Its open epoch is
|
|
3224
|
+
// the one its chain ends on; a chain it cannot describe has sealed an
|
|
3225
|
+
// epoch, which only a rotation does.
|
|
3226
|
+
let openEpoch;
|
|
2867
3227
|
try {
|
|
2868
|
-
|
|
2869
|
-
);
|
|
2870
|
-
summary.aliasesSet++;
|
|
3228
|
+
const described = await retryingImportStep(`Reading the epoch of ${documentId}`, () => client.exportDocumentChain(appId, documentId));
|
|
3229
|
+
openEpoch = Number(described?.epoch);
|
|
2871
3230
|
}
|
|
2872
|
-
catch {
|
|
2873
|
-
|
|
3231
|
+
catch (refusal) {
|
|
3232
|
+
if (refusal?.code !== "EXPORT_CHAIN_INCOMPLETE")
|
|
3233
|
+
throw refusal;
|
|
3234
|
+
return "rotated-epoch";
|
|
2874
3235
|
}
|
|
3236
|
+
if (openEpoch !== 1)
|
|
3237
|
+
return "rotated-epoch";
|
|
2875
3238
|
}
|
|
2876
3239
|
}
|
|
2877
|
-
|
|
3240
|
+
else {
|
|
3241
|
+
const exportedState = exportedStateOf(docDir);
|
|
3242
|
+
// An export with no state and no blobs can still carry a root's aliases.
|
|
3243
|
+
const exportsUserAliases = (Array.isArray(aliasOwner?.exported) ? aliasOwner.exported : [])
|
|
3244
|
+
.some((alias) => alias?.aliasScope === "user");
|
|
3245
|
+
if (!exportedState && exported.length === 0 && !exportsUserAliases)
|
|
3246
|
+
return null;
|
|
3247
|
+
const held = await retryingImportStep(`Reading the state of ${documentId}`, () => client.exportDocumentState(appId, documentId));
|
|
3248
|
+
const heldState = Buffer.from(String(held?.state ?? ""), "base64");
|
|
3249
|
+
if (isEmptyYjsUpdate(heldState)) {
|
|
3250
|
+
state = exportedState !== null;
|
|
3251
|
+
}
|
|
3252
|
+
else if (!exportedState || !holdsYjsUpdate(heldState, exportedState)) {
|
|
3253
|
+
return null;
|
|
3254
|
+
}
|
|
3255
|
+
}
|
|
3256
|
+
let blobs = [];
|
|
3257
|
+
if (exported.length > 0) {
|
|
3258
|
+
const listed = await listedBlobIds(client, appId, documentId);
|
|
3259
|
+
blobs = exported.filter((blob) => !listed.has(blob.blobId));
|
|
3260
|
+
}
|
|
3261
|
+
const aliases = aliasOwner
|
|
3262
|
+
? await findMissingAliases(client, appId, documentId, aliasOwner)
|
|
3263
|
+
: [];
|
|
3264
|
+
return state || chain || blobs.length > 0 || aliases.length > 0
|
|
3265
|
+
? { state, chain, blobs, aliases }
|
|
3266
|
+
: null;
|
|
3267
|
+
}
|
|
3268
|
+
/** The form the server stores an alias key in. */
|
|
3269
|
+
function normalizedAliasKey(aliasKey) {
|
|
3270
|
+
return String(aliasKey ?? "").trim().toLowerCase();
|
|
3271
|
+
}
|
|
3272
|
+
/**
|
|
3273
|
+
* The exported user aliases that do not point at the document for their
|
|
3274
|
+
* owner. One the owner holds for another document is only missing under
|
|
3275
|
+
* `--aliases overwrite`: under `skip` the restore leaves it where it is, so a
|
|
3276
|
+
* re-run would have nothing to write for it.
|
|
3277
|
+
*/
|
|
3278
|
+
async function findMissingAliases(client, appId, documentId, owner) {
|
|
3279
|
+
const wanted = (Array.isArray(owner.exported) ? owner.exported : []).filter((alias) => alias?.aliasScope === "user");
|
|
3280
|
+
if (wanted.length === 0)
|
|
3281
|
+
return [];
|
|
3282
|
+
const listed = await retryingImportStep(`Listing the aliases of ${documentId}`, () => client.listDocumentAliases(appId, documentId));
|
|
3283
|
+
const held = new Set((Array.isArray(listed) ? listed : (listed?.aliases ?? []))
|
|
3284
|
+
.filter((alias) => (alias?.scope || alias?.aliasScope) === "user" && alias?.userId === owner.userId)
|
|
3285
|
+
.map((alias) => normalizedAliasKey(alias.aliasKey)));
|
|
3286
|
+
const missing = [];
|
|
3287
|
+
for (const alias of wanted) {
|
|
3288
|
+
if (held.has(normalizedAliasKey(alias.aliasKey)))
|
|
3289
|
+
continue;
|
|
3290
|
+
if (owner.aliasMode === "skip") {
|
|
3291
|
+
const current = await retryingImportStep(`Reading alias ${alias.aliasKey}`, () => client.getDocumentAlias(appId, "user", alias.aliasKey, owner.userId));
|
|
3292
|
+
if (current)
|
|
3293
|
+
continue;
|
|
3294
|
+
}
|
|
3295
|
+
missing.push(alias);
|
|
3296
|
+
}
|
|
3297
|
+
return missing;
|
|
3298
|
+
}
|
|
3299
|
+
/** Every blob id a document lists, read to the last page. */
|
|
3300
|
+
async function listedBlobIds(client, appId, documentId) {
|
|
3301
|
+
const ids = new Set();
|
|
3302
|
+
let cursor;
|
|
3303
|
+
for (;;) {
|
|
3304
|
+
const page = await retryingImportStep(`Listing the blobs of ${documentId}`, () => client.listDocumentBlobs(appId, documentId, cursor ? { cursor } : undefined));
|
|
3305
|
+
const items = Array.isArray(page) ? page : (page?.items ?? []);
|
|
3306
|
+
for (const item of items) {
|
|
3307
|
+
if (item?.blobId)
|
|
3308
|
+
ids.add(String(item.blobId));
|
|
3309
|
+
}
|
|
3310
|
+
if (Array.isArray(page) || !page?.nextCursor)
|
|
3311
|
+
return ids;
|
|
3312
|
+
cursor = String(page.nextCursor);
|
|
3313
|
+
}
|
|
2878
3314
|
}
|
|
2879
3315
|
/**
|
|
2880
3316
|
* Restore a root-marked export into the target user's own root document
|
|
2881
|
-
* (#3135)
|
|
3317
|
+
* (#3135), by format (#4195): a legacy export's state is merged into a legacy
|
|
3318
|
+
* root, a large-document export's chain is installed into a large root.
|
|
2882
3319
|
*
|
|
2883
3320
|
* The exported id is never created and never looked up: it is the source
|
|
2884
3321
|
* user's root id, which means nothing in this app. Everything — the Yjs state,
|
|
@@ -2890,18 +3327,22 @@ async function importSingleDocument(client, appId, docDir, overwrite, aliasMode,
|
|
|
2890
3327
|
* holds survive, and a key both sides set resolves by Yjs's own conflict rule,
|
|
2891
3328
|
* which guarantees neither source-wins nor target-wins. That is why merging
|
|
2892
3329
|
* into an existing root needs `--overwrite` — the default must not quietly
|
|
2893
|
-
* rewrite a user's live preferences.
|
|
3330
|
+
* rewrite a user's live preferences. Without it, an existing legacy root is
|
|
3331
|
+
* only completed (#4148): one an earlier run of this import started and did
|
|
3332
|
+
* not finish gets what it lacks, exactly as a legacy document does. An
|
|
3333
|
+
* existing large root is skipped.
|
|
2894
3334
|
*/
|
|
2895
|
-
async function importRootDocument(client, appId, docDir, metadata, plan, overwrite, aliasMode, dryRun, summary) {
|
|
3335
|
+
async function importRootDocument(client, appId, docDir, metadata, plan, overwrite, aliasMode, dryRun, summary, progress) {
|
|
2896
3336
|
const exportedDocumentId = metadata.documentId || path.basename(docDir);
|
|
2897
3337
|
const wouldSkip = dryRun ? "[dry-run] Would skip" : "Skipping";
|
|
2898
|
-
const report = (action, targetRootDocId, reason) => {
|
|
3338
|
+
const report = (action, targetRootDocId, reason, targetFormat) => {
|
|
2899
3339
|
summary.rootDocuments.push({
|
|
2900
3340
|
exportedDocumentId,
|
|
2901
3341
|
owner: plan?.owner ?? "",
|
|
2902
3342
|
...(targetRootDocId ? { targetRootDocId } : {}),
|
|
2903
3343
|
action,
|
|
2904
3344
|
...(reason ? { reason } : {}),
|
|
3345
|
+
...(targetFormat === 2 ? { documentFormat: 2 } : {}),
|
|
2905
3346
|
});
|
|
2906
3347
|
};
|
|
2907
3348
|
if (!plan || plan.action === "refused") {
|
|
@@ -2913,31 +3354,64 @@ async function importRootDocument(client, appId, docDir, metadata, plan, overwri
|
|
|
2913
3354
|
report("refused", undefined, reason);
|
|
2914
3355
|
return;
|
|
2915
3356
|
}
|
|
3357
|
+
let targetRootDocId = plan.targetRootDocId;
|
|
3358
|
+
// What a root an earlier run started still lacks (#4148), when completing it.
|
|
3359
|
+
let gaps = null;
|
|
2916
3360
|
if (plan.action === "skip") {
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
|
|
2923
|
-
|
|
3361
|
+
// The root exists and `--overwrite` was not given. A legacy root an
|
|
3362
|
+
// earlier run started and did not finish is completed, by the same rule
|
|
3363
|
+
// as any legacy document: it holds no state, or already holds the
|
|
3364
|
+
// exported state. A root holding state of its own — its user's — is never
|
|
3365
|
+
// merged into. A large root is skipped as before: a chain is installed
|
|
3366
|
+
// only into a root that holds nothing, which `--overwrite` asks for.
|
|
3367
|
+
progress.exists = true;
|
|
3368
|
+
const found = plan.targetFormat === 2
|
|
3369
|
+
? null
|
|
3370
|
+
: await findImportGaps(client, appId, docDir, targetRootDocId, false, {
|
|
3371
|
+
userId: plan.targetUserId,
|
|
3372
|
+
aliasMode,
|
|
3373
|
+
exported: metadata.aliases,
|
|
3374
|
+
});
|
|
3375
|
+
if (!found || typeof found === "string") {
|
|
3376
|
+
const reason = plan.reason ||
|
|
3377
|
+
`the root document of ${plan.owner} already exists ` +
|
|
3378
|
+
`(${targetRootDocId}); use --overwrite to merge the exported ` +
|
|
3379
|
+
`content into it`;
|
|
3380
|
+
warn(`${wouldSkip} ${exportedDocumentId}: ${reason}`);
|
|
3381
|
+
summary.skipped++;
|
|
3382
|
+
report("skipped", targetRootDocId, reason, plan.targetFormat);
|
|
3383
|
+
return;
|
|
3384
|
+
}
|
|
3385
|
+
gaps = found;
|
|
2924
3386
|
}
|
|
3387
|
+
const install = plan.mode === "install";
|
|
2925
3388
|
if (dryRun) {
|
|
2926
|
-
if (
|
|
2927
|
-
info(`[dry-run] Would
|
|
2928
|
-
`
|
|
3389
|
+
if (gaps) {
|
|
3390
|
+
info(`[dry-run] Would complete root document export ${exportedDocumentId} ` +
|
|
3391
|
+
`in ${plan.owner}'s root document ${targetRootDocId}: it is missing ` +
|
|
3392
|
+
describeImportGaps(gaps));
|
|
3393
|
+
summary.completed++;
|
|
3394
|
+
report("completed", targetRootDocId);
|
|
3395
|
+
}
|
|
3396
|
+
else if (plan.action === "create") {
|
|
3397
|
+
info(install
|
|
3398
|
+
? `[dry-run] Would create the root document (large) for ` +
|
|
3399
|
+
`${plan.owner} and install root document export ` +
|
|
3400
|
+
`${exportedDocumentId} into it`
|
|
3401
|
+
: `[dry-run] Would create the root document for ${plan.owner} and ` +
|
|
3402
|
+
`apply root document export ${exportedDocumentId} to it`);
|
|
2929
3403
|
summary.created++;
|
|
2930
|
-
report("created");
|
|
3404
|
+
report("created", undefined, undefined, plan.targetFormat);
|
|
2931
3405
|
}
|
|
2932
3406
|
else {
|
|
2933
|
-
info(`[dry-run] Would
|
|
3407
|
+
info(`[dry-run] Would ${install ? "install" : "apply"} root document ` +
|
|
3408
|
+
`export ${exportedDocumentId} ${install ? "into" : "to"} ` +
|
|
2934
3409
|
`${plan.owner}'s root document ${plan.targetRootDocId}`);
|
|
2935
3410
|
summary.updated++;
|
|
2936
|
-
report("applied", plan.targetRootDocId);
|
|
3411
|
+
report("applied", plan.targetRootDocId, undefined, plan.targetFormat);
|
|
2937
3412
|
}
|
|
2938
3413
|
return;
|
|
2939
3414
|
}
|
|
2940
|
-
let targetRootDocId = plan.targetRootDocId;
|
|
2941
3415
|
let created = false;
|
|
2942
3416
|
if (plan.action === "create") {
|
|
2943
3417
|
// The server's own get-or-create, and the only place a root document is
|
|
@@ -2951,64 +3425,75 @@ async function importRootDocument(client, appId, docDir, metadata, plan, overwri
|
|
|
2951
3425
|
`for ${plan.owner}, so nothing was imported for it.`);
|
|
2952
3426
|
}
|
|
2953
3427
|
created = ensured.created === true;
|
|
3428
|
+
// The root the server returned is the truth, not the plan's prediction
|
|
3429
|
+
// (#4195): the app's setting can change, or a sign-in can mint the root,
|
|
3430
|
+
// between the plan and this call. Decided before the concurrent-sign-in
|
|
3431
|
+
// skip, so a root of the other format is refused, never offered to
|
|
3432
|
+
// --overwrite.
|
|
3433
|
+
const minted = decideRootImport({
|
|
3434
|
+
exportFormat: plan.exportFormat,
|
|
3435
|
+
existingRootFormat: rootDocumentFormatOf(ensured.documentFormat),
|
|
3436
|
+
predictedFormat: plan.targetFormat,
|
|
3437
|
+
appSetting: null,
|
|
3438
|
+
overwrite: true,
|
|
3439
|
+
owner: plan.owner,
|
|
3440
|
+
targetRootDocId,
|
|
3441
|
+
});
|
|
3442
|
+
if (minted.outcome === "refuse") {
|
|
3443
|
+
warn(`Skipping ${exportedDocumentId}: ${minted.reason}`);
|
|
3444
|
+
summary.skipped++;
|
|
3445
|
+
report("refused", targetRootDocId, minted.reason, minted.targetFormat);
|
|
3446
|
+
return;
|
|
3447
|
+
}
|
|
2954
3448
|
if (!created && !overwrite) {
|
|
2955
3449
|
const reason = `the root document of ${plan.owner} (${targetRootDocId}) was created ` +
|
|
2956
3450
|
`by a concurrent sign-in while this import was running; use ` +
|
|
2957
|
-
`--overwrite to merge the exported
|
|
3451
|
+
`--overwrite to ${install ? "install" : "merge"} the exported ` +
|
|
3452
|
+
`content into it`;
|
|
2958
3453
|
warn(`Skipping ${exportedDocumentId}: ${reason}`);
|
|
2959
3454
|
summary.skipped++;
|
|
2960
|
-
report("skipped", targetRootDocId, reason);
|
|
3455
|
+
report("skipped", targetRootDocId, reason, minted.targetFormat);
|
|
2961
3456
|
return;
|
|
2962
3457
|
}
|
|
2963
3458
|
}
|
|
3459
|
+
progress.exists = true;
|
|
2964
3460
|
// Only content moves. The target root keeps its own id, title, tags and
|
|
2965
3461
|
// permissions — the ensure path already granted the owner read-write.
|
|
2966
|
-
|
|
2967
|
-
|
|
2968
|
-
|
|
3462
|
+
if (install) {
|
|
3463
|
+
// A large root's content is its chain, installed — the server refuses a
|
|
3464
|
+
// root that already holds records or has rotated, as it does for any
|
|
3465
|
+
// large document (#4195).
|
|
3466
|
+
await importLargeDocumentChain(client, appId, docDir, targetRootDocId);
|
|
2969
3467
|
}
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
const blobIndex = JSON.parse(fs.readFileSync(blobIndexPath, "utf-8"));
|
|
2973
|
-
for (const blob of blobIndex) {
|
|
2974
|
-
const blobPath = path.join(docDir, "blobs", `${blob.blobId}.bin`);
|
|
2975
|
-
if (!fs.existsSync(blobPath))
|
|
2976
|
-
continue;
|
|
2977
|
-
// The blobId is preserved: a document's own references travel by blobId,
|
|
2978
|
-
// so they resolve inside the target root as they did in the source.
|
|
2979
|
-
await client.uploadBlob(appId, targetRootDocId, blob.blobId, fs.readFileSync(blobPath), {
|
|
2980
|
-
filename: blob.filename,
|
|
2981
|
-
contentType: blob.contentType,
|
|
2982
|
-
sha256: blob.sha256,
|
|
2983
|
-
});
|
|
2984
|
-
summary.blobsUploaded++;
|
|
2985
|
-
}
|
|
3468
|
+
else if (!gaps || gaps.state) {
|
|
3469
|
+
await importExportedState(client, appId, docDir, targetRootDocId);
|
|
2986
3470
|
}
|
|
3471
|
+
// The blobId is preserved: a document's own references travel by blobId,
|
|
3472
|
+
// so they resolve inside the target root as they did in the source.
|
|
3473
|
+
await uploadExportedBlobs(client, appId, targetRootDocId, docDir, gaps ? gaps.blobs : exportedBlobsOf(docDir), summary);
|
|
2987
3474
|
// A root document's user-scoped aliases belong to its owner by definition,
|
|
2988
3475
|
// so they are restored without `--owner` — the target user is already known.
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
}
|
|
2997
|
-
|
|
2998
|
-
summary.aliasesSkipped++;
|
|
2999
|
-
}
|
|
3000
|
-
}
|
|
3476
|
+
// A completion restores only the ones the root is missing.
|
|
3477
|
+
await restoreExportedAliases(client, appId, targetRootDocId, gaps ? { ...metadata, aliases: gaps.aliases } : metadata, plan.targetUserId, aliasMode, summary);
|
|
3478
|
+
const verb = install ? "installed" : "applied";
|
|
3479
|
+
if (gaps) {
|
|
3480
|
+
summary.completed++;
|
|
3481
|
+
report("completed", targetRootDocId);
|
|
3482
|
+
info(`Completed root document export ${exportedDocumentId} in ` +
|
|
3483
|
+
`${plan.owner}'s root document ${targetRootDocId}: imported ` +
|
|
3484
|
+
describeImportGaps(gaps));
|
|
3001
3485
|
}
|
|
3002
|
-
if (created) {
|
|
3486
|
+
else if (created) {
|
|
3003
3487
|
summary.created++;
|
|
3004
|
-
report("created", targetRootDocId);
|
|
3005
|
-
info(`Created root document ${targetRootDocId} for ${plan.owner} and
|
|
3006
|
-
`root document export ${exportedDocumentId} to it`);
|
|
3488
|
+
report("created", targetRootDocId, undefined, plan.targetFormat);
|
|
3489
|
+
info(`Created root document ${targetRootDocId} for ${plan.owner} and ${verb} ` +
|
|
3490
|
+
`root document export ${exportedDocumentId} ${install ? "into" : "to"} it`);
|
|
3007
3491
|
}
|
|
3008
3492
|
else {
|
|
3009
3493
|
summary.updated++;
|
|
3010
|
-
report("applied", targetRootDocId);
|
|
3011
|
-
info(
|
|
3494
|
+
report("applied", targetRootDocId, undefined, plan.targetFormat);
|
|
3495
|
+
info(`${install ? "Installed" : "Applied"} root document export ` +
|
|
3496
|
+
`${exportedDocumentId} ${install ? "into" : "to"} ${plan.owner}'s ` +
|
|
3012
3497
|
`root document ${targetRootDocId}`);
|
|
3013
3498
|
}
|
|
3014
3499
|
}
|