primitive-admin 1.2.0-alpha.1 → 1.2.0-alpha.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +105 -134
- package/assets/skill/skills/primitive-platform/SKILL.md +37 -4
- package/dist/bin/primitive.js +30 -35
- package/dist/bin/primitive.js.map +1 -1
- package/dist/src/commands/agent-sessions.d.ts +133 -0
- package/dist/src/commands/agent-sessions.js +792 -0
- package/dist/src/commands/agent-sessions.js.map +1 -0
- package/dist/src/commands/analytics.js +0 -148
- package/dist/src/commands/analytics.js.map +1 -1
- package/dist/src/commands/apps-children.d.ts +54 -0
- package/dist/src/commands/apps-children.js +631 -0
- package/dist/src/commands/apps-children.js.map +1 -0
- package/dist/src/commands/apps.d.ts +6 -0
- package/dist/src/commands/apps.js +131 -9
- package/dist/src/commands/apps.js.map +1 -1
- package/dist/src/commands/auth-sessions.d.ts +3 -3
- package/dist/src/commands/auth-sessions.js +117 -23
- package/dist/src/commands/auth-sessions.js.map +1 -1
- package/dist/src/commands/auth.js +460 -36
- package/dist/src/commands/auth.js.map +1 -1
- package/dist/src/commands/collections.js +316 -256
- package/dist/src/commands/collections.js.map +1 -1
- package/dist/src/commands/config.d.ts +1 -1
- package/dist/src/commands/config.js +60 -28
- package/dist/src/commands/config.js.map +1 -1
- package/dist/src/commands/connections.js +11 -12
- package/dist/src/commands/connections.js.map +1 -1
- package/dist/src/commands/database-type-configs.js +2 -76
- package/dist/src/commands/database-type-configs.js.map +1 -1
- package/dist/src/commands/databases.d.ts +9 -0
- package/dist/src/commands/databases.js +221 -457
- package/dist/src/commands/databases.js.map +1 -1
- package/dist/src/commands/db3-types.d.ts +15 -0
- package/dist/src/commands/db3-types.js +157 -0
- package/dist/src/commands/db3-types.js.map +1 -0
- package/dist/src/commands/db3.d.ts +12 -0
- package/dist/src/commands/db3.js +1052 -0
- package/dist/src/commands/db3.js.map +1 -0
- package/dist/src/commands/documents.d.ts +121 -2
- package/dist/src/commands/documents.js +1601 -1251
- package/dist/src/commands/documents.js.map +1 -1
- package/dist/src/commands/env.d.ts +9 -1
- package/dist/src/commands/env.js +164 -49
- package/dist/src/commands/env.js.map +1 -1
- package/dist/src/commands/function-trigger-commands.d.ts +28 -0
- package/dist/src/commands/function-trigger-commands.js +341 -0
- package/dist/src/commands/function-trigger-commands.js.map +1 -0
- package/dist/src/commands/functions.js +297 -24
- package/dist/src/commands/functions.js.map +1 -1
- package/dist/src/commands/groups.js +150 -49
- package/dist/src/commands/groups.js.map +1 -1
- package/dist/src/commands/guides.js +5 -5
- package/dist/src/commands/init.d.ts +7 -0
- package/dist/src/commands/init.js +23 -5
- package/dist/src/commands/init.js.map +1 -1
- package/dist/src/commands/integrations.js +14 -8
- package/dist/src/commands/integrations.js.map +1 -1
- package/dist/src/commands/metadata-category-configs.d.ts +2 -2
- package/dist/src/commands/metadata-category-configs.js +2 -2
- package/dist/src/commands/prompts.d.ts +7 -0
- package/dist/src/commands/prompts.js +48 -5
- package/dist/src/commands/prompts.js.map +1 -1
- package/dist/src/commands/scripts.d.ts +9 -17
- package/dist/src/commands/scripts.js +24 -479
- package/dist/src/commands/scripts.js.map +1 -1
- package/dist/src/commands/sync-app-settings.d.ts +36 -8
- package/dist/src/commands/sync-app-settings.js +109 -23
- package/dist/src/commands/sync-app-settings.js.map +1 -1
- package/dist/src/commands/sync.d.ts +360 -610
- package/dist/src/commands/sync.js +3151 -6697
- package/dist/src/commands/sync.js.map +1 -1
- package/dist/src/commands/tokens.js +3 -3
- package/dist/src/commands/tokens.js.map +1 -1
- package/dist/src/commands/users.js +77 -2
- package/dist/src/commands/users.js.map +1 -1
- package/dist/src/commands/workflows.d.ts +5 -83
- package/dist/src/commands/workflows.js +57 -2282
- package/dist/src/commands/workflows.js.map +1 -1
- package/dist/src/lib/aggregate-output.d.ts +25 -0
- package/dist/src/lib/aggregate-output.js +53 -0
- package/dist/src/lib/aggregate-output.js.map +1 -0
- package/dist/src/lib/api-client.d.ts +681 -551
- package/dist/src/lib/api-client.js +880 -814
- package/dist/src/lib/api-client.js.map +1 -1
- package/dist/src/lib/app-settings-descriptor.d.ts +47 -1
- package/dist/src/lib/app-settings-descriptor.js +114 -10
- package/dist/src/lib/app-settings-descriptor.js.map +1 -1
- package/dist/src/lib/auth-fixtures.d.ts +44 -0
- package/dist/src/lib/auth-fixtures.js +61 -0
- package/dist/src/lib/auth-fixtures.js.map +1 -0
- package/dist/src/lib/auth-flow.d.ts +41 -1
- package/dist/src/lib/auth-flow.js +70 -16
- package/dist/src/lib/auth-flow.js.map +1 -1
- package/dist/src/lib/block-layout.d.ts +12 -23
- package/dist/src/lib/block-layout.js +26 -92
- package/dist/src/lib/block-layout.js.map +1 -1
- package/dist/src/lib/block-selector.d.ts +2 -2
- package/dist/src/lib/block-selector.js +2 -2
- package/dist/src/lib/child-apps-local.d.ts +164 -0
- package/dist/src/lib/child-apps-local.js +258 -0
- package/dist/src/lib/child-apps-local.js.map +1 -0
- package/dist/src/lib/ci-session-request.d.ts +22 -0
- package/dist/src/lib/ci-session-request.js +51 -0
- package/dist/src/lib/ci-session-request.js.map +1 -0
- package/dist/src/lib/codegen-shared/generatedFiles.d.ts +10 -8
- package/dist/src/lib/codegen-shared/generatedFiles.js +21 -15
- package/dist/src/lib/codegen-shared/generatedFiles.js.map +1 -1
- package/dist/src/lib/codegen-shared/prettierStable.d.ts +1 -1
- package/dist/src/lib/codegen-shared/prettierStable.js +1 -1
- package/dist/src/lib/codegen-shared/resolveCodegenSourceDir.d.ts +2 -2
- package/dist/src/lib/codegen-shared/resolveCodegenSourceDir.js +2 -2
- package/dist/src/lib/collection-export.d.ts +76 -15
- package/dist/src/lib/collection-export.js +116 -23
- package/dist/src/lib/collection-export.js.map +1 -1
- package/dist/src/lib/collection-principal.d.ts +22 -0
- package/dist/src/lib/collection-principal.js +35 -0
- package/dist/src/lib/collection-principal.js.map +1 -0
- package/dist/src/lib/config-object-descriptor.d.ts +3 -22
- package/dist/src/lib/config-object-descriptor.js +56 -290
- package/dist/src/lib/config-object-descriptor.js.map +1 -1
- package/dist/src/lib/config-payload.js +2 -3
- package/dist/src/lib/config-payload.js.map +1 -1
- package/dist/src/lib/config-surface.d.ts +5 -7
- package/dist/src/lib/config-surface.js +18 -40
- package/dist/src/lib/config-surface.js.map +1 -1
- package/dist/src/lib/connections-list.d.ts +10 -0
- package/dist/src/lib/connections-list.js +18 -0
- package/dist/src/lib/connections-list.js.map +1 -0
- package/dist/src/lib/credentials-home.d.ts +58 -0
- package/dist/src/lib/credentials-home.js +204 -0
- package/dist/src/lib/credentials-home.js.map +1 -0
- package/dist/src/lib/credentials-lock.d.ts +88 -0
- package/dist/src/lib/credentials-lock.js +302 -0
- package/dist/src/lib/credentials-lock.js.map +1 -0
- package/dist/src/lib/credentials-store.d.ts +179 -24
- package/dist/src/lib/credentials-store.js +712 -77
- package/dist/src/lib/credentials-store.js.map +1 -1
- package/dist/src/lib/csv.d.ts +8 -9
- package/dist/src/lib/csv.js +3 -4
- package/dist/src/lib/csv.js.map +1 -1
- package/dist/src/lib/custodian.d.ts +8 -0
- package/dist/src/lib/custodian.js +18 -0
- package/dist/src/lib/custodian.js.map +1 -0
- package/dist/src/lib/db-codegen/dbGenerator.js +5 -16
- package/dist/src/lib/db-codegen/dbGenerator.js.map +1 -1
- package/dist/src/lib/db-codegen/dbNaming.d.ts +5 -13
- package/dist/src/lib/db-codegen/dbNaming.js +5 -17
- package/dist/src/lib/db-codegen/dbNaming.js.map +1 -1
- package/dist/src/lib/db-codegen/dbTemplates.d.ts +7 -56
- package/dist/src/lib/db-codegen/dbTemplates.js +25 -125
- package/dist/src/lib/db-codegen/dbTemplates.js.map +1 -1
- package/dist/src/lib/db-codegen/dbTsTypes.d.ts +2 -2
- package/dist/src/lib/db-codegen/dbTsTypes.js +2 -2
- package/dist/src/lib/db-codegen/dbTypeIR.d.ts +67 -2
- package/dist/src/lib/db-codegen/dbTypeIR.js +13 -2
- package/dist/src/lib/db-codegen/dbTypeIR.js.map +1 -1
- package/dist/src/lib/db3-export.d.ts +231 -0
- package/dist/src/lib/db3-export.js +510 -0
- package/dist/src/lib/db3-export.js.map +1 -0
- package/dist/src/lib/db3-index-output.d.ts +27 -0
- package/dist/src/lib/db3-index-output.js +49 -0
- package/dist/src/lib/db3-index-output.js.map +1 -0
- package/dist/src/lib/document-export-lookups.d.ts +31 -0
- package/dist/src/lib/document-export-lookups.js +53 -0
- package/dist/src/lib/document-export-lookups.js.map +1 -0
- package/dist/src/lib/document-export-permissions.d.ts +7 -5
- package/dist/src/lib/document-export-permissions.js +7 -5
- package/dist/src/lib/document-export-permissions.js.map +1 -1
- package/dist/src/lib/document-export-record-log.d.ts +57 -0
- package/dist/src/lib/document-export-record-log.js +215 -0
- package/dist/src/lib/document-export-record-log.js.map +1 -0
- package/dist/src/lib/document-ingest-artifact.d.ts +26 -2
- package/dist/src/lib/document-ingest-artifact.js +42 -8
- package/dist/src/lib/document-ingest-artifact.js.map +1 -1
- package/dist/src/lib/document-ingest-format2.d.ts +36 -0
- package/dist/src/lib/document-ingest-format2.js +78 -0
- package/dist/src/lib/document-ingest-format2.js.map +1 -0
- package/dist/src/lib/document-ingest-input.d.ts +9 -0
- package/dist/src/lib/document-ingest-input.js +28 -1
- package/dist/src/lib/document-ingest-input.js.map +1 -1
- package/dist/src/lib/document-ingest-rows.d.ts +16 -0
- package/dist/src/lib/document-ingest-rows.js +17 -0
- package/dist/src/lib/document-ingest-rows.js.map +1 -1
- package/dist/src/lib/document-ingest.d.ts +33 -1
- package/dist/src/lib/document-ingest.js +49 -0
- package/dist/src/lib/document-ingest.js.map +1 -1
- package/dist/src/lib/document-schema-rows.d.ts +42 -0
- package/dist/src/lib/document-schema-rows.js +53 -0
- package/dist/src/lib/document-schema-rows.js.map +1 -0
- package/dist/src/lib/env-resolver-core.d.ts +70 -2
- package/dist/src/lib/env-resolver-core.js +137 -20
- package/dist/src/lib/env-resolver-core.js.map +1 -1
- package/dist/src/lib/env-resolver.d.ts +14 -2
- package/dist/src/lib/env-resolver.js +32 -10
- package/dist/src/lib/env-resolver.js.map +1 -1
- package/dist/src/lib/function-authorize-check.d.ts +142 -0
- package/dist/src/lib/function-authorize-check.js +310 -0
- package/dist/src/lib/function-authorize-check.js.map +1 -0
- package/dist/src/lib/function-authorize-view.d.ts +15 -0
- package/dist/src/lib/function-authorize-view.js +50 -0
- package/dist/src/lib/function-authorize-view.js.map +1 -0
- package/dist/src/lib/function-bundle.d.ts +38 -18
- package/dist/src/lib/function-bundle.js +96 -35
- package/dist/src/lib/function-bundle.js.map +1 -1
- package/dist/src/lib/function-collect.d.ts +75 -12
- package/dist/src/lib/function-collect.js +427 -12
- package/dist/src/lib/function-collect.js.map +1 -1
- package/dist/src/lib/function-db-types.d.ts +29 -0
- package/dist/src/lib/function-db-types.js +117 -7
- package/dist/src/lib/function-db-types.js.map +1 -1
- package/dist/src/lib/function-document-types.d.ts +5 -3
- package/dist/src/lib/function-document-types.js +10 -14
- package/dist/src/lib/function-document-types.js.map +1 -1
- package/dist/src/lib/function-lint.d.ts +34 -0
- package/dist/src/lib/function-lint.js +112 -0
- package/dist/src/lib/function-lint.js.map +1 -0
- package/dist/src/lib/function-log-row.js +21 -2
- package/dist/src/lib/function-log-row.js.map +1 -1
- package/dist/src/lib/function-run.d.ts +16 -21
- package/dist/src/lib/function-run.js +37 -30
- package/dist/src/lib/function-run.js.map +1 -1
- package/dist/src/lib/function-schema-codegen.d.ts +1 -1
- package/dist/src/lib/function-schema-codegen.js +5 -5
- package/dist/src/lib/function-schema-codegen.js.map +1 -1
- package/dist/src/lib/function-source-roots.d.ts +67 -0
- package/dist/src/lib/function-source-roots.js +148 -0
- package/dist/src/lib/function-source-roots.js.map +1 -0
- package/dist/src/lib/function-sync.d.ts +108 -22
- package/dist/src/lib/function-sync.js +127 -39
- package/dist/src/lib/function-sync.js.map +1 -1
- package/dist/src/lib/function-trigger-listing.d.ts +11 -6
- package/dist/src/lib/function-trigger-listing.js +32 -11
- package/dist/src/lib/function-trigger-listing.js.map +1 -1
- package/dist/src/lib/function-typecheck.d.ts +37 -2
- package/dist/src/lib/function-typecheck.js +210 -28
- package/dist/src/lib/function-typecheck.js.map +1 -1
- package/dist/src/lib/function-versions.d.ts +2 -0
- package/dist/src/lib/function-versions.js +1 -0
- package/dist/src/lib/function-versions.js.map +1 -1
- package/dist/src/lib/generated-allowlist.js +0 -20
- package/dist/src/lib/generated-allowlist.js.map +1 -1
- package/dist/src/lib/generated-config-surfaces.d.ts +703 -203
- package/dist/src/lib/generated-config-surfaces.js +2270 -2014
- package/dist/src/lib/generated-config-surfaces.js.map +1 -1
- package/dist/src/lib/generated-sdk-types.d.ts +1 -1
- package/dist/src/lib/generated-sdk-types.js +1 -1
- package/dist/src/lib/generated-sdk-types.js.map +1 -1
- package/dist/src/lib/init-adopt.js +12 -3
- package/dist/src/lib/init-adopt.js.map +1 -1
- package/dist/src/lib/init-login-note.d.ts +13 -0
- package/dist/src/lib/init-login-note.js +22 -0
- package/dist/src/lib/init-login-note.js.map +1 -0
- package/dist/src/lib/json-value.d.ts +6 -0
- package/dist/src/lib/json-value.js +13 -0
- package/dist/src/lib/json-value.js.map +1 -0
- package/dist/src/lib/load-routes.d.ts +33 -0
- package/dist/src/lib/load-routes.js +35 -0
- package/dist/src/lib/load-routes.js.map +1 -0
- package/dist/src/lib/local-state.d.ts +38 -7
- package/dist/src/lib/local-state.js +172 -49
- package/dist/src/lib/local-state.js.map +1 -1
- package/dist/src/lib/local-test-cases.d.ts +5 -2
- package/dist/src/lib/local-test-cases.js +7 -4
- package/dist/src/lib/local-test-cases.js.map +1 -1
- package/dist/src/lib/log-inspection.d.ts +42 -4
- package/dist/src/lib/log-inspection.js +37 -5
- package/dist/src/lib/log-inspection.js.map +1 -1
- package/dist/src/lib/login-exchange-check.d.ts +18 -0
- package/dist/src/lib/login-exchange-check.js +27 -0
- package/dist/src/lib/login-exchange-check.js.map +1 -0
- package/dist/src/lib/login-scope-lines.d.ts +25 -0
- package/dist/src/lib/login-scope-lines.js +45 -0
- package/dist/src/lib/login-scope-lines.js.map +1 -0
- package/dist/src/lib/logout-admin-session.d.ts +2 -1
- package/dist/src/lib/logout-admin-session.js +7 -1
- package/dist/src/lib/logout-admin-session.js.map +1 -1
- package/dist/src/lib/object-status-filter.d.ts +9 -1
- package/dist/src/lib/object-status-filter.js +9 -1
- package/dist/src/lib/object-status-filter.js.map +1 -1
- package/dist/src/lib/output.js +2 -0
- package/dist/src/lib/output.js.map +1 -1
- package/dist/src/lib/paginate.d.ts +0 -5
- package/dist/src/lib/paginate.js +0 -5
- package/dist/src/lib/paginate.js.map +1 -1
- package/dist/src/lib/pin-refusal-hint.d.ts +12 -0
- package/dist/src/lib/pin-refusal-hint.js +35 -0
- package/dist/src/lib/pin-refusal-hint.js.map +1 -0
- package/dist/src/lib/primitive-env-plugin.d.ts +99 -0
- package/dist/src/lib/primitive-env-plugin.js +408 -0
- package/dist/src/lib/primitive-env-plugin.js.map +1 -0
- package/dist/src/lib/project-config.d.ts +7 -0
- package/dist/src/lib/project-config.js +22 -1
- package/dist/src/lib/project-config.js.map +1 -1
- package/dist/src/lib/project-login-apps.d.ts +8 -0
- package/dist/src/lib/project-login-apps.js +27 -0
- package/dist/src/lib/project-login-apps.js.map +1 -0
- package/dist/src/lib/prompt-schema-codegen.d.ts +10 -3
- package/dist/src/lib/prompt-schema-codegen.js +52 -11
- package/dist/src/lib/prompt-schema-codegen.js.map +1 -1
- package/dist/src/lib/pull-write.d.ts +31 -0
- package/dist/src/lib/pull-write.js +49 -0
- package/dist/src/lib/pull-write.js.map +1 -0
- package/dist/src/lib/push-failure-report.d.ts +45 -0
- package/dist/src/lib/push-failure-report.js +57 -0
- package/dist/src/lib/push-failure-report.js.map +1 -0
- package/dist/src/lib/records-verbs.d.ts +69 -0
- package/dist/src/lib/records-verbs.js +40 -0
- package/dist/src/lib/records-verbs.js.map +1 -0
- package/dist/src/lib/refresh-admin-credentials.d.ts +7 -0
- package/dist/src/lib/refresh-admin-credentials.js +16 -4
- package/dist/src/lib/refresh-admin-credentials.js.map +1 -1
- package/dist/src/lib/root-import-decision.d.ts +65 -0
- package/dist/src/lib/root-import-decision.js +109 -0
- package/dist/src/lib/root-import-decision.js.map +1 -0
- package/dist/src/lib/run-status.d.ts +1 -1
- package/dist/src/lib/run-status.generated.d.ts +2 -2
- package/dist/src/lib/run-status.generated.js +3 -1
- package/dist/src/lib/run-status.generated.js.map +1 -1
- package/dist/src/lib/run-status.js +1 -1
- package/dist/src/lib/schema-codegen/generated-schema-descriptor.d.ts +129 -0
- package/dist/src/lib/schema-codegen/generated-schema-descriptor.js +269 -0
- package/dist/src/lib/schema-codegen/generated-schema-descriptor.js.map +1 -0
- package/dist/src/lib/schema-codegen/naming.d.ts +35 -0
- package/dist/src/lib/schema-codegen/naming.js +83 -0
- package/dist/src/lib/schema-codegen/naming.js.map +1 -0
- package/dist/src/lib/schema-codegen/schemaToTs.d.ts +82 -0
- package/dist/src/lib/schema-codegen/schemaToTs.js +305 -0
- package/dist/src/lib/schema-codegen/schemaToTs.js.map +1 -0
- package/dist/src/lib/scope-request.d.ts +84 -0
- package/dist/src/lib/scope-request.js +235 -0
- package/dist/src/lib/scope-request.js.map +1 -0
- package/dist/src/lib/session-id.d.ts +9 -0
- package/dist/src/lib/session-id.js +28 -0
- package/dist/src/lib/session-id.js.map +1 -0
- package/dist/src/lib/snapshot-build-rows.d.ts +21 -0
- package/dist/src/lib/snapshot-build-rows.js +22 -0
- package/dist/src/lib/snapshot-build-rows.js.map +1 -1
- package/dist/src/lib/snapshot-build.js +8 -3
- package/dist/src/lib/snapshot-build.js.map +1 -1
- package/dist/src/lib/snapshots.d.ts +17 -1
- package/dist/src/lib/snapshots.js +50 -2
- package/dist/src/lib/snapshots.js.map +1 -1
- package/dist/src/lib/swift-codegen/agentGenerator.d.ts +6 -2
- package/dist/src/lib/swift-codegen/agentGenerator.js +57 -5
- package/dist/src/lib/swift-codegen/agentGenerator.js.map +1 -1
- package/dist/src/lib/swift-codegen/banners.d.ts +34 -7
- package/dist/src/lib/swift-codegen/banners.js +37 -7
- package/dist/src/lib/swift-codegen/banners.js.map +1 -1
- package/dist/src/lib/swift-codegen/dbGenerator.d.ts +4 -3
- package/dist/src/lib/swift-codegen/dbGenerator.js +14 -117
- package/dist/src/lib/swift-codegen/dbGenerator.js.map +1 -1
- package/dist/src/lib/swift-codegen/functionGenerator.d.ts +3 -3
- package/dist/src/lib/swift-codegen/functionGenerator.js +23 -18
- package/dist/src/lib/swift-codegen/functionGenerator.js.map +1 -1
- package/dist/src/lib/swift-codegen/generator.d.ts +5 -5
- package/dist/src/lib/swift-codegen/generator.js +35 -40
- package/dist/src/lib/swift-codegen/generator.js.map +1 -1
- package/dist/src/lib/swift-codegen/schemaToSwift.d.ts +3 -3
- package/dist/src/lib/swift-codegen/schemaToSwift.js +4 -4
- package/dist/src/lib/swift-codegen/schemaToSwift.js.map +1 -1
- package/dist/src/lib/swift-codegen/swiftKeys.d.ts +19 -0
- package/dist/src/lib/swift-codegen/swiftKeys.js +31 -0
- package/dist/src/lib/swift-codegen/swiftKeys.js.map +1 -0
- package/dist/src/lib/sync-paths.d.ts +35 -0
- package/dist/src/lib/sync-paths.js +57 -5
- package/dist/src/lib/sync-paths.js.map +1 -1
- package/dist/src/lib/sync-resource-types.d.ts +26 -26
- package/dist/src/lib/sync-resource-types.js +61 -189
- package/dist/src/lib/sync-resource-types.js.map +1 -1
- package/dist/src/lib/sync-selectors.d.ts +5 -1
- package/dist/src/lib/sync-selectors.js +23 -13
- package/dist/src/lib/sync-selectors.js.map +1 -1
- package/dist/src/lib/test-case-file-names.d.ts +3 -5
- package/dist/src/lib/test-case-file-names.js +3 -5
- package/dist/src/lib/test-case-file-names.js.map +1 -1
- package/dist/src/lib/toml-database-config.d.ts +15 -75
- package/dist/src/lib/toml-database-config.js +17 -306
- package/dist/src/lib/toml-database-config.js.map +1 -1
- package/dist/src/lib/transient-retry.d.ts +43 -0
- package/dist/src/lib/transient-retry.js +94 -0
- package/dist/src/lib/transient-retry.js.map +1 -0
- package/dist/src/lib/vars-override.d.ts +105 -0
- package/dist/src/lib/vars-override.js +143 -0
- package/dist/src/lib/vars-override.js.map +1 -0
- package/dist/src/lib/webhook-capture.d.ts +43 -0
- package/dist/src/lib/webhook-capture.js +91 -0
- package/dist/src/lib/webhook-capture.js.map +1 -0
- package/dist/src/lib/webhook-deliver.d.ts +18 -20
- package/dist/src/lib/webhook-deliver.js +34 -89
- package/dist/src/lib/webhook-deliver.js.map +1 -1
- package/dist/src/lib/webhook-test-run.d.ts +32 -0
- package/dist/src/lib/webhook-test-run.js +108 -0
- package/dist/src/lib/webhook-test-run.js.map +1 -0
- package/dist/src/lib/workflow-codegen/generator.d.ts +7 -7
- package/dist/src/lib/workflow-codegen/generator.js +16 -23
- package/dist/src/lib/workflow-codegen/generator.js.map +1 -1
- package/dist/src/lib/workflow-codegen/invokerIR.d.ts +6 -12
- package/dist/src/lib/workflow-codegen/invokerIR.js +4 -5
- package/dist/src/lib/workflow-codegen/invokerIR.js.map +1 -1
- package/dist/src/lib/workflow-toml-validator.d.ts +2 -0
- package/dist/src/lib/workflow-toml-validator.js +28 -1
- package/dist/src/lib/workflow-toml-validator.js.map +1 -1
- package/dist/src/types/index.d.ts +30 -37
- package/dist/src/validators.d.ts +7 -55
- package/dist/src/validators.js +7 -53
- package/dist/src/validators.js.map +1 -1
- package/dist/src/vite.d.ts +17 -0
- package/dist/src/vite.js +18 -0
- package/dist/src/vite.js.map +1 -0
- package/package.json +20 -12
|
@@ -1,27 +1,32 @@
|
|
|
1
|
-
import { snapshotBuildRows } from "../lib/snapshot-build-rows.js";
|
|
1
|
+
import { segmentRows, snapshotBuildRows } from "../lib/snapshot-build-rows.js";
|
|
2
|
+
import { exportRecordLogDocument } from "../lib/document-export-record-log.js";
|
|
2
3
|
import { driveSnapshotBuild } from "../lib/snapshot-build.js";
|
|
3
4
|
import { ingestClientRows, ingestSessionRows, } from "../lib/document-ingest-rows.js";
|
|
4
5
|
import { ApiClient } from "../lib/api-client.js";
|
|
6
|
+
import { documentSchemaRows } from "../lib/document-schema-rows.js";
|
|
5
7
|
import { resolveAppId } from "../lib/config.js";
|
|
6
8
|
import { success, error, info, formatTable, formatId, formatDate, json, keyValue, result as printResult, warn, flushOutput, } from "../lib/output.js";
|
|
9
|
+
import { renderAggregateResult } from "../lib/aggregate-output.js";
|
|
7
10
|
import { confirmPrompt } from "../lib/confirm-prompt.js";
|
|
8
11
|
import { resolveOwnerUserId } from "../lib/resolve-owner.js";
|
|
9
|
-
import { pageCursorOption, pageLimitOption, parsePageLimit, printEmptyPage, printPageHint, } from "../lib/list-options.js";
|
|
12
|
+
import { pageCursorOption, pageLimitOption, parsePageLimit, printEmptyPage, printPageHint, quoteCursorArg, } from "../lib/list-options.js";
|
|
10
13
|
import { normalizeCliListEnvelope, wholeListEnvelope, } from "../lib/paginate.js";
|
|
11
14
|
import { parseDataOption } from "../lib/data-input.js";
|
|
12
15
|
import { parseFilterOptions } from "../lib/record-filter.js";
|
|
13
|
-
import { buildPermissionsExport } from "../lib/document-export-permissions.js";
|
|
14
|
-
import {
|
|
15
|
-
import { validateSnapshotManifest } from "js-bao";
|
|
16
|
-
import { auditSnapshotChain, } from "../lib/snapshot-audit.js";
|
|
17
|
-
import { createAuditApiSource } from "../lib/snapshot-audit-source.js";
|
|
18
|
-
import { createAuditLocalStore, createChainFingerprintReader, } from "../lib/snapshot-audit-store.js";
|
|
16
|
+
import { buildPermissionsExport, PENDING_INVITATIONS_ABSENT_STATUSES, } from "../lib/document-export-permissions.js";
|
|
17
|
+
import { readExportLookup } from "../lib/document-export-lookups.js";
|
|
19
18
|
import { discoverIngestInput, } from "../lib/document-ingest-input.js";
|
|
20
19
|
import { buildIngestArtifact, ingestSchemaFromIntrospection, } from "../lib/document-ingest-artifact.js";
|
|
21
|
-
import { driveDocumentIngest, summariseIngestPlan, assertIngestConfirmable, } from "../lib/document-ingest.js";
|
|
20
|
+
import { driveDocumentIngest, summariseIngestPlan, assertIngestConfirmable, assertIngestRefreshKeepsRefused, ingestCreateBody, parseIngestFlags, } from "../lib/document-ingest.js";
|
|
21
|
+
import { withTransientRetry } from "../lib/transient-retry.js";
|
|
22
|
+
import { recordLogSchemaToToml } from "js-bao";
|
|
23
|
+
import { DOCUMENT_ROUTES } from "../lib/records-verbs.js";
|
|
24
|
+
import { DOCUMENT_LOAD_ROUTES } from "../lib/load-routes.js";
|
|
22
25
|
import { ulid } from "ulid";
|
|
26
|
+
import * as Y from "yjs";
|
|
23
27
|
import * as fs from "fs";
|
|
24
28
|
import * as path from "path";
|
|
29
|
+
import { decideRootImport, rootDocumentFormatOf, } from "../lib/root-import-decision.js";
|
|
25
30
|
/**
|
|
26
31
|
* One bulk-load session as rows (#3434, #3435, #3598).
|
|
27
32
|
*
|
|
@@ -35,6 +40,17 @@ function printIngestSession(session) {
|
|
|
35
40
|
printResult(label, value);
|
|
36
41
|
}
|
|
37
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* One refused document call, as one line for an operator: the server's
|
|
45
|
+
* sentence and, when it sent one, the stable code a script branches on — a
|
|
46
|
+
* large document's refusal names `DOCUMENT_FORMAT_UNSUPPORTED` on every verb
|
|
47
|
+
* that reaches its data (#4313).
|
|
48
|
+
*/
|
|
49
|
+
export function describeDocumentFailure(err) {
|
|
50
|
+
const message = (typeof err?.message === "string" && err.message) || String(err);
|
|
51
|
+
const code = typeof err?.code === "string" && err.code ? err.code : undefined;
|
|
52
|
+
return code && !message.includes(code) ? `${message} [${code}]` : message;
|
|
53
|
+
}
|
|
38
54
|
/**
|
|
39
55
|
* One refused records call, as one line for an operator (#3764).
|
|
40
56
|
*
|
|
@@ -53,7 +69,8 @@ export function describeRecordsFailure(err) {
|
|
|
53
69
|
return `${message}${code ? ` [${code}]` : ""}${status !== undefined ? ` (${status})` : ""}`;
|
|
54
70
|
}
|
|
55
71
|
/**
|
|
56
|
-
* Report a document's format (#2816) — but only when it is a large
|
|
72
|
+
* Report a document's format (#2816) — but only when it is a large (2) or a
|
|
73
|
+
* record-log (3, #4306) document.
|
|
57
74
|
*
|
|
58
75
|
* The format is chosen at creation and never migrated, so an operator has to
|
|
59
76
|
* be able to read it back: `--large` that the server ignored would otherwise
|
|
@@ -62,9 +79,13 @@ export function describeRecordsFailure(err) {
|
|
|
62
79
|
* stored row spells the format (absent, null or 1).
|
|
63
80
|
*/
|
|
64
81
|
function printDocumentFormat(doc) {
|
|
65
|
-
|
|
82
|
+
const format = Number(doc?.documentFormat);
|
|
83
|
+
if (format === 2) {
|
|
66
84
|
keyValue("Format", "2 (large document)");
|
|
67
85
|
}
|
|
86
|
+
else if (format === 3) {
|
|
87
|
+
keyValue("Format", "3 (record log)");
|
|
88
|
+
}
|
|
68
89
|
}
|
|
69
90
|
/**
|
|
70
91
|
* The `--document-format` an operator stated, or the refusal (#3764, E10).
|
|
@@ -72,24 +93,65 @@ function printDocumentFormat(doc) {
|
|
|
72
93
|
* Refused HERE rather than by the server, because an operator CAN act on it
|
|
73
94
|
* (principle 6) and a round trip to be told what they typed wrong is a round
|
|
74
95
|
* trip they do not need to spend. Commander hands the raw string over, so the
|
|
75
|
-
* digits are what is read; an unset flag states nothing.
|
|
96
|
+
* digits are what is read; an unset flag states nothing. `flag` names the
|
|
97
|
+
* option in the refusal: `documents create` reads its `--format` here too.
|
|
98
|
+
*
|
|
99
|
+
* `2` — a large document, which the platform no longer serves — is refused
|
|
100
|
+
* with `DOCUMENT_FORMAT_UNSUPPORTED` named, as the server would refuse it.
|
|
76
101
|
*/
|
|
77
|
-
export function parseDocumentFormatOption(value) {
|
|
102
|
+
export function parseDocumentFormatOption(value, flag = "--document-format") {
|
|
78
103
|
if (value === undefined || value === null)
|
|
79
104
|
return undefined;
|
|
80
105
|
const text = String(value).trim();
|
|
81
106
|
if (text === "1")
|
|
82
107
|
return 1;
|
|
83
|
-
if (text === "
|
|
84
|
-
return
|
|
85
|
-
|
|
86
|
-
`
|
|
108
|
+
if (text === "3")
|
|
109
|
+
return 3;
|
|
110
|
+
if (text === "2") {
|
|
111
|
+
throw new Error(`documentFormat 2 (large document) is no longer served; use ${flag} 3 (DOCUMENT_FORMAT_UNSUPPORTED)`);
|
|
112
|
+
}
|
|
113
|
+
throw new Error(`Invalid ${flag} "${String(value)}". Expected 1 (an ordinary ` +
|
|
114
|
+
`document) or 3 (a record-log document).`);
|
|
87
115
|
}
|
|
88
|
-
/**
|
|
89
|
-
|
|
90
|
-
|
|
116
|
+
/**
|
|
117
|
+
* The `documentFormat` a `documents create` sends, from its two flags.
|
|
118
|
+
*
|
|
119
|
+
* `--format` is read by {@link parseDocumentFormatOption}, so a value outside
|
|
120
|
+
* 1 and 3 is refused before any request with the same sentence every records
|
|
121
|
+
* verb uses, and 2 with `DOCUMENT_FORMAT_UNSUPPORTED`. `--large` is
|
|
122
|
+
* `--format 3`: alone it sends 3, beside `--format 3` it changes nothing, and
|
|
123
|
+
* beside `--format 1` the two disagree and the command is refused rather than
|
|
124
|
+
* guessing which one was meant. Neither flag sends nothing, so the server's
|
|
125
|
+
* default stands.
|
|
126
|
+
*
|
|
127
|
+
* @param flags - The raw `--format` value and whether `--large` was given.
|
|
128
|
+
* @returns The format to send, or `undefined` to send none.
|
|
129
|
+
* @throws Error when `--format` is not 1 or 3, or disagrees with `--large`.
|
|
130
|
+
*/
|
|
131
|
+
export function documentFormatForCreate(flags) {
|
|
132
|
+
const format = parseDocumentFormatOption(flags.format, "--format");
|
|
133
|
+
if (!flags.large)
|
|
134
|
+
return format;
|
|
135
|
+
if (format !== undefined && format !== 3) {
|
|
136
|
+
throw new Error("--large is --format 3; pass one of them");
|
|
137
|
+
}
|
|
138
|
+
return 3;
|
|
139
|
+
}
|
|
140
|
+
/** The `--document-format <1|3>` flag, said once for every records verb. */
|
|
141
|
+
const DOCUMENT_FORMAT_FLAG = "--document-format <1|3>";
|
|
142
|
+
const DOCUMENT_FORMAT_HELP = "The document format you expect: 1 (ordinary) or 3 (record log). A disagreement " +
|
|
91
143
|
"with the platform's own answer is refused with DOCUMENT_FORMAT_MISMATCH " +
|
|
92
144
|
"rather than answered out of the wrong tables";
|
|
145
|
+
/**
|
|
146
|
+
* One row of `documents inspect --refused`, as the label and value it prints:
|
|
147
|
+
* the code, a constraint refusal's constraint and holder, and the reason a
|
|
148
|
+
* record-log rule refusal carries (#4621).
|
|
149
|
+
*/
|
|
150
|
+
export function refusedRowLine(row) {
|
|
151
|
+
const conflicting = row.conflicting ? ` (held by ${row.conflicting.model} ${row.conflicting.id})` : "";
|
|
152
|
+
const reason = row.reason ? `: ${row.reason}` : "";
|
|
153
|
+
return [` ${row.model} ${row.id}`, `${row.code}${row.constraint ? ` ${row.constraint}` : ""}${conflicting}${reason}`];
|
|
154
|
+
}
|
|
93
155
|
/**
|
|
94
156
|
* What `documents records bulk` reports when the write is refused (#3619).
|
|
95
157
|
*
|
|
@@ -124,6 +186,32 @@ export function describeBulkFailure(err) {
|
|
|
124
186
|
},
|
|
125
187
|
};
|
|
126
188
|
}
|
|
189
|
+
/**
|
|
190
|
+
* The `--data-file` help of `documents records bulk`: the op shapes the route
|
|
191
|
+
* accepts, including the `upsert` action addressed by a declared unique
|
|
192
|
+
* constraint (#3750).
|
|
193
|
+
*/
|
|
194
|
+
export const BULK_DATA_FILE_HELP = "JSON file with { operations: [...] } or a bare operations array. " +
|
|
195
|
+
"Each op is { model, action: create|patch|delete, id, data, precondition? } " +
|
|
196
|
+
"— `data` holds the record fields and is required (non-empty) on " +
|
|
197
|
+
"create/patch, not allowed on delete; a create `id` must be a 26-char " +
|
|
198
|
+
"uppercase Crockford ULID. An upsert op is { model, action: upsert, " +
|
|
199
|
+
"constraint, data, id?, precondition? } — `constraint` names a unique " +
|
|
200
|
+
"constraint the model declares and `data` carries its fields; the record " +
|
|
201
|
+
"holding that key is merged into, or created (at `id` when given)";
|
|
202
|
+
/**
|
|
203
|
+
* The success line of `documents records bulk` (#3750): the blob's counts,
|
|
204
|
+
* with the upsert ops and how many of them created their record. An answer
|
|
205
|
+
* without `upserted` (an older server) reads as no upserts.
|
|
206
|
+
*/
|
|
207
|
+
export function describeBulkResult(result) {
|
|
208
|
+
const upserted = Array.isArray(result.upserted) ? result.upserted : [];
|
|
209
|
+
const created = upserted.filter((entry) => entry.created).length;
|
|
210
|
+
return (`Applied ${result.applied} operation(s): ` +
|
|
211
|
+
`${result.added.length} added, ${result.updated.length} updated, ` +
|
|
212
|
+
`${result.deleted} deleted, ${upserted.length} upserted` +
|
|
213
|
+
`${upserted.length > 0 ? ` (${created} created)` : ""}.`);
|
|
214
|
+
}
|
|
127
215
|
/**
|
|
128
216
|
* A document's tags as a list, from whatever the wire handed back (#3644).
|
|
129
217
|
*
|
|
@@ -181,6 +269,7 @@ Examples:
|
|
|
181
269
|
$ primitive documents get <document-id>
|
|
182
270
|
$ primitive documents create "My Doc"
|
|
183
271
|
$ primitive documents create "My Doc" --owner user@example.com
|
|
272
|
+
$ primitive documents create "Ledger" --format 3
|
|
184
273
|
$ primitive documents delete <document-id> -y
|
|
185
274
|
$ primitive documents permissions list <document-id>
|
|
186
275
|
$ primitive documents permissions grant <document-id> --user-id <user-id> --permission reader
|
|
@@ -188,6 +277,7 @@ Examples:
|
|
|
188
277
|
$ primitive documents records models <document-id>
|
|
189
278
|
$ primitive documents records describe <document-id> <model-name>
|
|
190
279
|
$ primitive documents records query <document-id> <model-name> --filter '{"status":"open"}'
|
|
280
|
+
$ primitive documents records query <document-id> <model-name> --sort '{"createdAt":-1}' --limit 50
|
|
191
281
|
$ primitive documents records count <document-id> <model-name>
|
|
192
282
|
$ primitive documents records save <document-id> <model-name> --data '{"qty":10}'
|
|
193
283
|
$ primitive documents records patch <document-id> <model-name> <record-id> --data '{"qty":11}'
|
|
@@ -303,15 +393,17 @@ Examples:
|
|
|
303
393
|
.argument("<title>", "Document title")
|
|
304
394
|
.option("--app <app-id>", "App ID")
|
|
305
395
|
.option("--owner <userId-or-email>", "User ID or email to own the document (super-admin or assigned-console-admin tokens only)")
|
|
306
|
-
.option("--
|
|
396
|
+
.option("--format <n>", "Document format: 1 (ordinary) or 3 (record log). Chosen at creation and never changed")
|
|
397
|
+
.option("--large", "Create a record-log document (format 3), for documents that outgrow a single in-memory Y.Doc. The same as --format 3")
|
|
307
398
|
.option("--json", "Output as JSON")
|
|
308
399
|
.addHelpText("after", `
|
|
309
400
|
Ownership:
|
|
310
401
|
An app-user token — member, admin, or owner — always creates the document
|
|
311
402
|
owned by the caller: the server ignores --owner for those tokens.
|
|
312
403
|
A super-admin token, or a console-admin token assigned to this app, acts
|
|
313
|
-
through an admin shadow app user — without --owner that shadow user owns
|
|
314
|
-
document
|
|
404
|
+
through an admin shadow app user — without --owner that shadow user owns an
|
|
405
|
+
ordinary document and the app holds a record-log document (--format 3),
|
|
406
|
+
which no member owns; with --owner the named user does.
|
|
315
407
|
A console admin not assigned to this app has no access to the app at all.
|
|
316
408
|
|
|
317
409
|
An email given to --owner is resolved to a user id before anything is
|
|
@@ -319,6 +411,20 @@ Ownership:
|
|
|
319
411
|
document.
|
|
320
412
|
`)
|
|
321
413
|
.action(async (title, options) => {
|
|
414
|
+
// Decided before anything else: a format the operator mistyped, one that
|
|
415
|
+
// disagrees with --large, or 2 (a large document, no longer served) is
|
|
416
|
+
// refused without a request (#4308, #4313).
|
|
417
|
+
let documentFormat;
|
|
418
|
+
try {
|
|
419
|
+
documentFormat = documentFormatForCreate({
|
|
420
|
+
format: options.format,
|
|
421
|
+
large: options.large === true,
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
catch (err) {
|
|
425
|
+
error(err.message);
|
|
426
|
+
process.exit(1);
|
|
427
|
+
}
|
|
322
428
|
const resolvedAppId = resolveAppId(undefined, options);
|
|
323
429
|
const client = new ApiClient();
|
|
324
430
|
let ownerUserId;
|
|
@@ -335,9 +441,9 @@ Ownership:
|
|
|
335
441
|
const result = await client.createDocument(resolvedAppId, {
|
|
336
442
|
title,
|
|
337
443
|
...(ownerUserId ? { createdBy: ownerUserId } : {}),
|
|
338
|
-
// Opt-in only: without --large nothing is sent, so the
|
|
339
|
-
// created in the legacy format exactly as before (#2816).
|
|
340
|
-
...(
|
|
444
|
+
// Opt-in only: without --format or --large nothing is sent, so the
|
|
445
|
+
// document is created in the legacy format exactly as before (#2816).
|
|
446
|
+
...(documentFormat !== undefined ? { documentFormat } : {}),
|
|
341
447
|
});
|
|
342
448
|
if (options.json) {
|
|
343
449
|
json(result);
|
|
@@ -347,7 +453,7 @@ Ownership:
|
|
|
347
453
|
keyValue("Document ID", result.documentId);
|
|
348
454
|
keyValue("Title", result.title);
|
|
349
455
|
keyValue("Owner", result.createdBy);
|
|
350
|
-
// Read back from the RESPONSE, not from
|
|
456
|
+
// Read back from the RESPONSE, not from the flags: what the server
|
|
351
457
|
// created is what the document will be for the rest of its life.
|
|
352
458
|
printDocumentFormat(result);
|
|
353
459
|
}
|
|
@@ -407,7 +513,7 @@ Permissions (enforced by the server, not the CLI):
|
|
|
407
513
|
catch (err) {
|
|
408
514
|
// Server refusals (404 not found, 403 root document / owner-only, the
|
|
409
515
|
// non-owner fail-closed 503) are printed verbatim, never swallowed.
|
|
410
|
-
error(err
|
|
516
|
+
error(describeDocumentFailure(err));
|
|
411
517
|
process.exit(1);
|
|
412
518
|
}
|
|
413
519
|
});
|
|
@@ -549,29 +655,7 @@ Permissions (enforced by the server, not the CLI):
|
|
|
549
655
|
.option("--app <app-id>", "App ID")
|
|
550
656
|
.option("--json", "Output as JSON")
|
|
551
657
|
.action(async (documentId, options) => {
|
|
552
|
-
|
|
553
|
-
const client = new ApiClient();
|
|
554
|
-
try {
|
|
555
|
-
const result = await client.getDocumentSchema(resolvedAppId, documentId);
|
|
556
|
-
const models = result?.modelNames ?? Object.keys(result?.schema?.models ?? {});
|
|
557
|
-
if (options.json) {
|
|
558
|
-
json(models);
|
|
559
|
-
return;
|
|
560
|
-
}
|
|
561
|
-
if (models.length === 0) {
|
|
562
|
-
info("No models found.");
|
|
563
|
-
return;
|
|
564
|
-
}
|
|
565
|
-
for (const m of models) {
|
|
566
|
-
console.log(` ${m}`);
|
|
567
|
-
}
|
|
568
|
-
}
|
|
569
|
-
catch (err) {
|
|
570
|
-
// #3764 — the server's sentence, the stable code and the status, so a
|
|
571
|
-
// refused records call is scriptable rather than only readable.
|
|
572
|
-
error(describeRecordsFailure(err));
|
|
573
|
-
process.exit(1);
|
|
574
|
-
}
|
|
658
|
+
await modelsAction(DOCUMENT_ROUTES, documentId, options);
|
|
575
659
|
});
|
|
576
660
|
// Describe a model's fields and indexes
|
|
577
661
|
records
|
|
@@ -582,49 +666,7 @@ Permissions (enforced by the server, not the CLI):
|
|
|
582
666
|
.option("--app <app-id>", "App ID")
|
|
583
667
|
.option("--json", "Output as JSON")
|
|
584
668
|
.action(async (documentId, modelName, options) => {
|
|
585
|
-
|
|
586
|
-
const client = new ApiClient();
|
|
587
|
-
try {
|
|
588
|
-
const result = await client.getDocumentSchema(resolvedAppId, documentId);
|
|
589
|
-
const model = result?.schema?.models?.[modelName];
|
|
590
|
-
if (!model) {
|
|
591
|
-
if (options.json) {
|
|
592
|
-
json(null);
|
|
593
|
-
return;
|
|
594
|
-
}
|
|
595
|
-
info(`Model '${modelName}' not found in this document.`);
|
|
596
|
-
return;
|
|
597
|
-
}
|
|
598
|
-
const fieldsObj = model.fields ?? {};
|
|
599
|
-
const rows = Object.entries(fieldsObj).map(([name, meta]) => ({
|
|
600
|
-
field: name,
|
|
601
|
-
type: meta?.type ?? "unknown",
|
|
602
|
-
indexed: meta?.indexed ? "yes" : "",
|
|
603
|
-
unique: meta?.unique ? "yes" : "",
|
|
604
|
-
required: meta?.required ? "yes" : "",
|
|
605
|
-
}));
|
|
606
|
-
if (options.json) {
|
|
607
|
-
json({ fields: fieldsObj, constraints: model.constraints ?? {} });
|
|
608
|
-
return;
|
|
609
|
-
}
|
|
610
|
-
if (rows.length === 0) {
|
|
611
|
-
info("No fields detected for this model.");
|
|
612
|
-
return;
|
|
613
|
-
}
|
|
614
|
-
console.log(formatTable(rows, [
|
|
615
|
-
{ header: "FIELD", key: "field", flex: true },
|
|
616
|
-
{ header: "TYPE", key: "type" },
|
|
617
|
-
{ header: "INDEXED", key: "indexed" },
|
|
618
|
-
{ header: "UNIQUE", key: "unique" },
|
|
619
|
-
{ header: "REQUIRED", key: "required" },
|
|
620
|
-
]));
|
|
621
|
-
}
|
|
622
|
-
catch (err) {
|
|
623
|
-
// #3764 — the server's sentence, the stable code and the status, so a
|
|
624
|
-
// refused records call is scriptable rather than only readable.
|
|
625
|
-
error(describeRecordsFailure(err));
|
|
626
|
-
process.exit(1);
|
|
627
|
-
}
|
|
669
|
+
await describeAction(DOCUMENT_ROUTES, documentId, modelName, options);
|
|
628
670
|
});
|
|
629
671
|
// Query records in a document model
|
|
630
672
|
records
|
|
@@ -635,8 +677,12 @@ Permissions (enforced by the server, not the CLI):
|
|
|
635
677
|
.option("--app <app-id>", "App ID")
|
|
636
678
|
.option("--filter <json>", "Filter as JSON (e.g. '{\"status\":\"open\"}')")
|
|
637
679
|
.option("--filter-file <path>", "Read filter from a JSON or TOML file")
|
|
638
|
-
.option("--
|
|
639
|
-
|
|
680
|
+
.option("--sort <json>", "Sort as JSON, e.g. '{\"createdAt\":-1}'; 1 ascending, -1 descending, key order is precedence; repeat it with --cursor")
|
|
681
|
+
// Forwarded as typed: the server refuses a value outside 1–10000 with a
|
|
682
|
+
// sentence naming the maximum, which a CLI-side parse would hide.
|
|
683
|
+
.option("--limit <n>", "Records per page (default 100, max 10000)")
|
|
684
|
+
.option("--cursor <cursor>", "Pagination cursor from a previous query (valid only with the same --sort)")
|
|
685
|
+
.option("--include <json>", "Related records as a JSON array of include specs, e.g. '[{\"model\":\"customer\",\"type\":\"refersTo\",\"sourceField\":\"customerId\"}]'; answered on a db3 as _related")
|
|
640
686
|
.option(DOCUMENT_FORMAT_FLAG, DOCUMENT_FORMAT_HELP)
|
|
641
687
|
.option("--json", "Output as JSON")
|
|
642
688
|
.action(async (documentId, modelName, options) => {
|
|
@@ -651,52 +697,7 @@ Permissions (enforced by the server, not the CLI):
|
|
|
651
697
|
error(describeRecordsFailure(err));
|
|
652
698
|
process.exit(1);
|
|
653
699
|
}
|
|
654
|
-
|
|
655
|
-
const client = new ApiClient();
|
|
656
|
-
try {
|
|
657
|
-
const result = await client.queryDocumentRecords(resolvedAppId, documentId, modelName, { filter, limit: options.limit, cursor: options.cursor, documentFormat });
|
|
658
|
-
if (options.json) {
|
|
659
|
-
json(result);
|
|
660
|
-
return;
|
|
661
|
-
}
|
|
662
|
-
const items = result.items;
|
|
663
|
-
if (items.length === 0) {
|
|
664
|
-
info("No records found.");
|
|
665
|
-
return;
|
|
666
|
-
}
|
|
667
|
-
const allKeys = new Set();
|
|
668
|
-
for (const rec of items) {
|
|
669
|
-
for (const key of Object.keys(rec))
|
|
670
|
-
allKeys.add(key);
|
|
671
|
-
}
|
|
672
|
-
const orderedKeys = [];
|
|
673
|
-
if (allKeys.has("id")) {
|
|
674
|
-
orderedKeys.push("id");
|
|
675
|
-
allKeys.delete("id");
|
|
676
|
-
}
|
|
677
|
-
orderedKeys.push(...[...allKeys].sort());
|
|
678
|
-
const columns = orderedKeys.map((key) => ({
|
|
679
|
-
header: key.toUpperCase(),
|
|
680
|
-
key,
|
|
681
|
-
format: (v) => {
|
|
682
|
-
if (v === null || v === undefined)
|
|
683
|
-
return "—";
|
|
684
|
-
if (typeof v === "object")
|
|
685
|
-
return JSON.stringify(v);
|
|
686
|
-
return String(v);
|
|
687
|
-
},
|
|
688
|
-
}));
|
|
689
|
-
console.log(formatTable(items, columns));
|
|
690
|
-
if (result.nextCursor) {
|
|
691
|
-
info(`More results available. Use --cursor ${result.nextCursor}`);
|
|
692
|
-
}
|
|
693
|
-
}
|
|
694
|
-
catch (err) {
|
|
695
|
-
// #3764 — the server's sentence, the stable code and the status, so a
|
|
696
|
-
// refused records call is scriptable rather than only readable.
|
|
697
|
-
error(describeRecordsFailure(err));
|
|
698
|
-
process.exit(1);
|
|
699
|
-
}
|
|
700
|
+
await queryRecordsAction(DOCUMENT_ROUTES, resolvedAppId, documentId, modelName, options, documentFormat);
|
|
700
701
|
});
|
|
701
702
|
// Get a single record by id (mirrors `databases records get`, issue #2357).
|
|
702
703
|
records
|
|
@@ -778,28 +779,12 @@ Permissions (enforced by the server, not the CLI):
|
|
|
778
779
|
error(describeRecordsFailure(err));
|
|
779
780
|
process.exit(1);
|
|
780
781
|
}
|
|
781
|
-
|
|
782
|
-
const client = new ApiClient();
|
|
783
|
-
try {
|
|
784
|
-
const result = await client.countDocumentRecords(resolvedAppId, documentId, modelName, { filter, documentFormat });
|
|
785
|
-
if (options.json) {
|
|
786
|
-
json(result);
|
|
787
|
-
return;
|
|
788
|
-
}
|
|
789
|
-
console.log(` ${result.count}`);
|
|
790
|
-
}
|
|
791
|
-
catch (err) {
|
|
792
|
-
// #3764 — the server's sentence, the stable code and the status, so a
|
|
793
|
-
// refused records call is scriptable rather than only readable.
|
|
794
|
-
error(describeRecordsFailure(err));
|
|
795
|
-
process.exit(1);
|
|
796
|
-
}
|
|
782
|
+
await countRecordsAction(DOCUMENT_ROUTES, resolvedAppId, documentId, modelName, options, documentFormat);
|
|
797
783
|
});
|
|
798
784
|
// Aggregate records in a document model (issue #2437). Mirrors
|
|
799
785
|
// `databases records aggregate` flag for flag and rendering for rendering —
|
|
800
786
|
// including the deliberate `keyValue()` for the ungrouped scalar, so the two
|
|
801
787
|
// twins keep printing the same thing.
|
|
802
|
-
const AGGREGATE_OPS = ["count", "sum", "avg", "min", "max"];
|
|
803
788
|
records
|
|
804
789
|
.command("aggregate")
|
|
805
790
|
.description("Aggregate records in a document model (count/sum/avg/min/max, optional group-by)")
|
|
@@ -825,70 +810,7 @@ Permissions (enforced by the server, not the CLI):
|
|
|
825
810
|
error(describeRecordsFailure(err));
|
|
826
811
|
process.exit(1);
|
|
827
812
|
}
|
|
828
|
-
|
|
829
|
-
error(`Invalid --op "${options.op}". Expected one of: ${AGGREGATE_OPS.join(", ")}.`);
|
|
830
|
-
process.exit(1);
|
|
831
|
-
}
|
|
832
|
-
const operation = { type: options.op };
|
|
833
|
-
if (options.op === "count") {
|
|
834
|
-
if (options.field) {
|
|
835
|
-
error("--field is not valid for --op count.");
|
|
836
|
-
process.exit(1);
|
|
837
|
-
}
|
|
838
|
-
}
|
|
839
|
-
else {
|
|
840
|
-
if (!options.field) {
|
|
841
|
-
error(`--op ${options.op} requires --field.`);
|
|
842
|
-
process.exit(1);
|
|
843
|
-
}
|
|
844
|
-
operation.field = options.field;
|
|
845
|
-
}
|
|
846
|
-
const filter = parseFilterOptions(options);
|
|
847
|
-
const groupBy = options.groupBy || [];
|
|
848
|
-
const client = new ApiClient();
|
|
849
|
-
try {
|
|
850
|
-
const response = await client.aggregateDocumentRecords(resolvedAppId, documentId, modelName, { operations: [operation], groupBy, filter }, { documentFormat });
|
|
851
|
-
if (options.json) {
|
|
852
|
-
json(response);
|
|
853
|
-
return;
|
|
854
|
-
}
|
|
855
|
-
const aggregated = response?.result;
|
|
856
|
-
// Same labelling rule as the databases twin: the field name is
|
|
857
|
-
// reproduced verbatim, never case-folded (issue #2357).
|
|
858
|
-
const valueLabel = options.op === "count"
|
|
859
|
-
? "COUNT"
|
|
860
|
-
: `${options.op.toUpperCase()}(${options.field})`;
|
|
861
|
-
const scalarLabel = options.op === "count"
|
|
862
|
-
? "count"
|
|
863
|
-
: `${options.op.toLowerCase()}(${options.field})`;
|
|
864
|
-
if (groupBy.length === 0) {
|
|
865
|
-
const value = (aggregated ?? {})[options.op === "count" ? "count" : `${options.op}_${options.field}`];
|
|
866
|
-
keyValue(scalarLabel, value === null || value === undefined ? "—" : String(value));
|
|
867
|
-
return;
|
|
868
|
-
}
|
|
869
|
-
if (aggregated &&
|
|
870
|
-
typeof aggregated === "object" &&
|
|
871
|
-
!Array.isArray(aggregated) &&
|
|
872
|
-
Object.values(aggregated).every((v) => typeof v !== "object" || v === null)) {
|
|
873
|
-
const rows = Object.entries(aggregated).map(([group, value]) => ({
|
|
874
|
-
group,
|
|
875
|
-
value: value === null || value === undefined ? "—" : String(value),
|
|
876
|
-
}));
|
|
877
|
-
console.log(formatTable(rows, [
|
|
878
|
-
{ header: "GROUP", key: "group" },
|
|
879
|
-
{ header: valueLabel, key: "value" },
|
|
880
|
-
]));
|
|
881
|
-
}
|
|
882
|
-
else {
|
|
883
|
-
console.log(JSON.stringify(aggregated ?? {}, null, 2));
|
|
884
|
-
}
|
|
885
|
-
}
|
|
886
|
-
catch (err) {
|
|
887
|
-
// #3764 — the server's sentence, the stable code and the status, so a
|
|
888
|
-
// refused records call is scriptable rather than only readable.
|
|
889
|
-
error(describeRecordsFailure(err));
|
|
890
|
-
process.exit(1);
|
|
891
|
-
}
|
|
813
|
+
await aggregateRecordsAction(DOCUMENT_ROUTES, resolvedAppId, documentId, modelName, options, documentFormat);
|
|
892
814
|
});
|
|
893
815
|
// ---- Record writes (#1964 Phase 4) ----
|
|
894
816
|
// Server-side writes through the document facade: Yjs-first, CRDT-safe,
|
|
@@ -924,23 +846,7 @@ Permissions (enforced by the server, not the CLI):
|
|
|
924
846
|
error(describeRecordsFailure(err));
|
|
925
847
|
process.exit(1);
|
|
926
848
|
}
|
|
927
|
-
|
|
928
|
-
const writeOptions = options.upsertOn ? { upsertOn: options.upsertOn } : undefined;
|
|
929
|
-
const client = new ApiClient();
|
|
930
|
-
try {
|
|
931
|
-
const result = await client.saveDocumentRecord(resolvedAppId, documentId, modelName, { id: recordId, data, ...(writeOptions ? { options: writeOptions } : {}) }, { documentFormat });
|
|
932
|
-
if (options.json) {
|
|
933
|
-
json(result);
|
|
934
|
-
return;
|
|
935
|
-
}
|
|
936
|
-
success(`Record saved: ${result.record?.id ?? recordId}`);
|
|
937
|
-
}
|
|
938
|
-
catch (err) {
|
|
939
|
-
// #3764 — the server's sentence, the stable code and the status, so a
|
|
940
|
-
// refused records call is scriptable rather than only readable.
|
|
941
|
-
error(describeRecordsFailure(err));
|
|
942
|
-
process.exit(1);
|
|
943
|
-
}
|
|
849
|
+
await saveRecordAction(DOCUMENT_ROUTES, resolvedAppId, documentId, modelName, data, options, documentFormat);
|
|
944
850
|
});
|
|
945
851
|
// Merge fields into an existing record
|
|
946
852
|
records
|
|
@@ -967,21 +873,7 @@ Permissions (enforced by the server, not the CLI):
|
|
|
967
873
|
error(describeRecordsFailure(err));
|
|
968
874
|
process.exit(1);
|
|
969
875
|
}
|
|
970
|
-
|
|
971
|
-
try {
|
|
972
|
-
const result = await client.patchDocumentRecord(resolvedAppId, documentId, modelName, recordId, { data }, { documentFormat });
|
|
973
|
-
if (options.json) {
|
|
974
|
-
json(result);
|
|
975
|
-
return;
|
|
976
|
-
}
|
|
977
|
-
success(`Record patched: ${recordId}`);
|
|
978
|
-
}
|
|
979
|
-
catch (err) {
|
|
980
|
-
// #3764 — the server's sentence, the stable code and the status, so a
|
|
981
|
-
// refused records call is scriptable rather than only readable.
|
|
982
|
-
error(describeRecordsFailure(err));
|
|
983
|
-
process.exit(1);
|
|
984
|
-
}
|
|
876
|
+
await patchRecordAction(DOCUMENT_ROUTES, resolvedAppId, documentId, modelName, recordId, data, options, documentFormat);
|
|
985
877
|
});
|
|
986
878
|
// Delete a record
|
|
987
879
|
records
|
|
@@ -1006,41 +898,14 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1006
898
|
error(describeRecordsFailure(err));
|
|
1007
899
|
process.exit(1);
|
|
1008
900
|
}
|
|
1009
|
-
|
|
1010
|
-
const confirmed = await confirmPrompt(`Delete record ${recordId} from ${modelName} in document ${documentId}?`);
|
|
1011
|
-
if (!confirmed) {
|
|
1012
|
-
info("Cancelled.");
|
|
1013
|
-
return;
|
|
1014
|
-
}
|
|
1015
|
-
}
|
|
1016
|
-
const client = new ApiClient();
|
|
1017
|
-
try {
|
|
1018
|
-
const result = await client.deleteDocumentRecord(resolvedAppId, documentId, modelName, recordId, { documentFormat });
|
|
1019
|
-
if (options.json) {
|
|
1020
|
-
json(result);
|
|
1021
|
-
return;
|
|
1022
|
-
}
|
|
1023
|
-
// A delete of a missing record is a silent no-op server-side, so
|
|
1024
|
-
// this reports the request outcome, not record existence.
|
|
1025
|
-
success(`Record deleted: ${recordId}`);
|
|
1026
|
-
}
|
|
1027
|
-
catch (err) {
|
|
1028
|
-
// #3764 — the server's sentence, the stable code and the status, so a
|
|
1029
|
-
// refused records call is scriptable rather than only readable.
|
|
1030
|
-
error(describeRecordsFailure(err));
|
|
1031
|
-
process.exit(1);
|
|
1032
|
-
}
|
|
901
|
+
await deleteRecordAction(DOCUMENT_ROUTES, resolvedAppId, documentId, modelName, recordId, options, documentFormat);
|
|
1033
902
|
});
|
|
1034
903
|
// Apply an atomic multi-model operations blob
|
|
1035
904
|
records
|
|
1036
905
|
.command("bulk")
|
|
1037
|
-
.description("Apply an atomic create/patch/delete operations blob to a document")
|
|
906
|
+
.description("Apply an atomic create/patch/delete/upsert operations blob to a document")
|
|
1038
907
|
.argument("<document-id>", "Document ID")
|
|
1039
|
-
.requiredOption("--data-file <file>",
|
|
1040
|
-
"Each op is { model, action: create|patch|delete, id, data, precondition? } " +
|
|
1041
|
-
"— `data` holds the record fields and is required (non-empty) on " +
|
|
1042
|
-
"create/patch, not allowed on delete; a create `id` must be a 26-char " +
|
|
1043
|
-
"uppercase Crockford ULID")
|
|
908
|
+
.requiredOption("--data-file <file>", BULK_DATA_FILE_HELP)
|
|
1044
909
|
.option("--app <app-id>", "App ID")
|
|
1045
910
|
.option(DOCUMENT_FORMAT_FLAG, DOCUMENT_FORMAT_HELP)
|
|
1046
911
|
.option("-y, --yes", "Skip confirmation prompt")
|
|
@@ -1056,40 +921,7 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1056
921
|
error(describeRecordsFailure(err));
|
|
1057
922
|
process.exit(1);
|
|
1058
923
|
}
|
|
1059
|
-
|
|
1060
|
-
allowArray: true,
|
|
1061
|
-
});
|
|
1062
|
-
const operations = Array.isArray(parsed) ? parsed : parsed?.operations;
|
|
1063
|
-
if (!Array.isArray(operations) || operations.length === 0) {
|
|
1064
|
-
error("--data-file must contain { operations: [...] } or a non-empty operations array.");
|
|
1065
|
-
process.exit(1);
|
|
1066
|
-
}
|
|
1067
|
-
const resolvedAppId = resolveAppId(undefined, options);
|
|
1068
|
-
if (!options.yes) {
|
|
1069
|
-
const confirmed = await confirmPrompt(`Apply ${operations.length} operation(s) atomically to document ${documentId}?`);
|
|
1070
|
-
if (!confirmed) {
|
|
1071
|
-
info("Cancelled.");
|
|
1072
|
-
return;
|
|
1073
|
-
}
|
|
1074
|
-
}
|
|
1075
|
-
const client = new ApiClient();
|
|
1076
|
-
try {
|
|
1077
|
-
const result = await client.bulkDocumentRecords(resolvedAppId, documentId, operations, { documentFormat });
|
|
1078
|
-
if (options.json) {
|
|
1079
|
-
json(result);
|
|
1080
|
-
return;
|
|
1081
|
-
}
|
|
1082
|
-
success(`Applied ${result.applied} operation(s): ` +
|
|
1083
|
-
`${result.added.length} added, ${result.updated.length} updated, ${result.deleted} deleted.`);
|
|
1084
|
-
}
|
|
1085
|
-
catch (err) {
|
|
1086
|
-
const report = describeBulkFailure(err);
|
|
1087
|
-
if (options.json)
|
|
1088
|
-
json(report.envelope);
|
|
1089
|
-
else
|
|
1090
|
-
error(report.line);
|
|
1091
|
-
process.exit(1);
|
|
1092
|
-
}
|
|
924
|
+
await bulkRecordsAction(DOCUMENT_ROUTES, documentId, options, documentFormat);
|
|
1093
925
|
});
|
|
1094
926
|
// Dump a document's records grouped by model (CLI-side composition)
|
|
1095
927
|
documents
|
|
@@ -1128,37 +960,45 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1128
960
|
console.log(output);
|
|
1129
961
|
}
|
|
1130
962
|
catch (err) {
|
|
1131
|
-
error(err
|
|
963
|
+
error(describeDocumentFailure(err));
|
|
1132
964
|
process.exit(1);
|
|
1133
965
|
}
|
|
1134
966
|
});
|
|
1135
967
|
// Document statistics (platform-vocabulary projection)
|
|
1136
968
|
documents
|
|
1137
969
|
.command("stats")
|
|
1138
|
-
.description("Show a document's record/model/blob counts and
|
|
970
|
+
.description("Show a document's record/model/blob counts and size")
|
|
1139
971
|
.argument("<document-id>", "Document ID")
|
|
1140
972
|
.option("--app <app-id>", "App ID")
|
|
1141
973
|
.option("--json", "Output as JSON")
|
|
1142
974
|
.action(async (documentId, options) => {
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
975
|
+
await statsAction(DOCUMENT_ROUTES, documentId, options);
|
|
976
|
+
});
|
|
977
|
+
// #4623 — a document's models and fields as the room declared and observed
|
|
978
|
+
// them (`GET .../schema`): a table, the route's JSON, or a schema.toml the
|
|
979
|
+
// client's `schemaToml` option and js-bao-codegen-v2 accept.
|
|
980
|
+
documents
|
|
981
|
+
.command("schema")
|
|
982
|
+
.description("Show a document's models and fields as declared and observed, or export them as a schema.toml")
|
|
983
|
+
.argument("<document-id>", "Document ID")
|
|
984
|
+
.option("--app <app-id>", "App ID")
|
|
985
|
+
.option("--toml", "Print a schema.toml the client's schemaToml option and js-bao-codegen-v2 accept")
|
|
986
|
+
.option("--output <file>", "With --toml: write the TOML to this file (overwriting it) instead of stdout")
|
|
987
|
+
.option("--json", "Output the /schema answer as JSON")
|
|
988
|
+
.action(async (documentId, options) => {
|
|
989
|
+
await schemaAction(DOCUMENT_ROUTES, documentId, options);
|
|
990
|
+
});
|
|
991
|
+
// #4306 — a record-log document's room: the inspection route's payload.
|
|
992
|
+
documents
|
|
993
|
+
.command("inspect")
|
|
994
|
+
.description("Show a record-log document's room: epochs, log number, floors, writers, the bulk load in progress and checkpoint")
|
|
995
|
+
.argument("<document-id>", "Document ID")
|
|
996
|
+
.option("--app <app-id>", "App ID")
|
|
997
|
+
.option("--refused <session-id>", "Print every row that bulk load refused, with its code, instead")
|
|
998
|
+
.option("--limit <n>", "With --refused: rows per page read (1–100)", "100")
|
|
999
|
+
.option("--json", "Output as JSON")
|
|
1000
|
+
.action(async (documentId, options) => {
|
|
1001
|
+
await inspectAction(DOCUMENT_ROUTES, documentId, options);
|
|
1162
1002
|
});
|
|
1163
1003
|
// The action verb (#3435, criterion 9): a directory of records becomes a
|
|
1164
1004
|
// chunked, digested, uploaded artifact and one atomic swap of the
|
|
@@ -1174,132 +1014,23 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1174
1014
|
.argument("<document-id>", "Document ID")
|
|
1175
1015
|
.requiredOption("--input <dir>", "Directory of <model>.ndjson[.gz] files, or a `documents export` output")
|
|
1176
1016
|
.option("--app <app-id>", "App ID")
|
|
1017
|
+
.option("--reserve-epoch <when>", "Record-log (format 3) documents: reserve the load's epoch at session open (open, the default) or at commit")
|
|
1018
|
+
.option("--mode <mode>", "Record-log (format 3) documents: merge the rows (merge, the default), or refresh: also delete every record of the input's models that no row names")
|
|
1177
1019
|
.option("-y, --yes", "Skip the confirmation prompt")
|
|
1178
1020
|
.option("--no-wait", "Exit once the session is committed")
|
|
1179
1021
|
.option("--timeout <seconds>", "Give up waiting after this many seconds")
|
|
1180
1022
|
.option("--json", "Output as JSON")
|
|
1181
1023
|
.action(async (documentId, options) => {
|
|
1182
|
-
|
|
1183
|
-
const client = new ApiClient();
|
|
1184
|
-
let plan = null;
|
|
1185
|
-
// The exit code is carried out of here rather than taken inside: a
|
|
1186
|
-
// `process.exit` does not unwind, so ending the process in the try block
|
|
1187
|
-
// would skip the cleanup in the finally and leave the compressed
|
|
1188
|
-
// artifact — up to the whole input, gzipped — in the OS temp directory
|
|
1189
|
-
// on every run, successful or not (finding 3435-FROZEN-05).
|
|
1190
|
-
const run = async () => {
|
|
1191
|
-
// The preflight, in this order and before a byte is read from disk:
|
|
1192
|
-
// the schema read says whether this caller may see the document's
|
|
1193
|
-
// declarations at all and hands over the ones the lines are checked
|
|
1194
|
-
// against, and the session reader says whether this is a large
|
|
1195
|
-
// document. Both are READS on the arm the ingest routes admit, so a
|
|
1196
|
-
// caller who would be refused halfway is refused at once, and neither
|
|
1197
|
-
// opens a session.
|
|
1198
|
-
const introspection = await client.getDocumentSchema(resolvedAppId, documentId);
|
|
1199
|
-
await client.listDocumentIngests(resolvedAppId, documentId, { limit: 1 });
|
|
1200
|
-
const schema = ingestSchemaFromIntrospection(introspection);
|
|
1201
|
-
const input = discoverIngestInput(options.input, documentId);
|
|
1202
|
-
for (const ignored of input.ignored) {
|
|
1203
|
-
info(`Ignoring ${ignored}: a bulk load reads the snapshot only.`);
|
|
1204
|
-
}
|
|
1205
|
-
// #3598 — reading, checking, sorting, cutting and gzipping is this
|
|
1206
|
-
// side's biggest phase on a large input, and it used to be invisible
|
|
1207
|
-
// inside one wall-clock number that also covered the platform's work.
|
|
1208
|
-
const buildStartedAt = Date.now();
|
|
1209
|
-
plan = await buildIngestArtifact({ input, schema });
|
|
1210
|
-
const buildMs = Date.now() - buildStartedAt;
|
|
1211
|
-
info(`Built artifact in ${(buildMs / 1000).toFixed(1)}s.`);
|
|
1212
|
-
for (const line of summariseIngestPlan(plan))
|
|
1213
|
-
info(line);
|
|
1214
|
-
assertIngestConfirmable({
|
|
1215
|
-
yes: Boolean(options.yes),
|
|
1216
|
-
isTTY: Boolean(process.stdin.isTTY),
|
|
1217
|
-
});
|
|
1218
|
-
if (!options.yes) {
|
|
1219
|
-
const confirmed = await confirmPrompt(`Bulk-load ${plan.rows} records into ${documentId}? ` +
|
|
1220
|
-
"This replaces records in a live document and cannot be undone.");
|
|
1221
|
-
if (!confirmed) {
|
|
1222
|
-
info("Nothing was uploaded.");
|
|
1223
|
-
return 1;
|
|
1224
|
-
}
|
|
1225
|
-
}
|
|
1226
|
-
// Ctrl-C is a decision the driver makes differently depending on
|
|
1227
|
-
// where it lands, so it is handed the signal rather than killing the
|
|
1228
|
-
// process: mid-upload the session is aborted, mid-poll it is left
|
|
1229
|
-
// running, because by then the document is already being changed.
|
|
1230
|
-
const interrupt = new AbortController();
|
|
1231
|
-
const onSignal = () => interrupt.abort();
|
|
1232
|
-
process.on("SIGINT", onSignal);
|
|
1233
|
-
let outcome;
|
|
1234
|
-
try {
|
|
1235
|
-
outcome = await driveDocumentIngest({
|
|
1236
|
-
client: {
|
|
1237
|
-
createIngest: (id) => client.createDocumentIngest(resolvedAppId, id),
|
|
1238
|
-
uploadIngestChunk: (id, sessionId, chunk) => client.uploadDocumentIngestChunk(resolvedAppId, id, sessionId, chunk),
|
|
1239
|
-
commitIngest: (id, sessionId) => client.commitDocumentIngest(resolvedAppId, id, sessionId),
|
|
1240
|
-
abortIngest: (id, sessionId) => client.abortDocumentIngest(resolvedAppId, id, sessionId),
|
|
1241
|
-
getIngest: (id, sessionId) => client.getDocumentIngest(resolvedAppId, id, sessionId),
|
|
1242
|
-
},
|
|
1243
|
-
documentId,
|
|
1244
|
-
plan,
|
|
1245
|
-
wait: options.wait !== false,
|
|
1246
|
-
timeoutMs: options.timeout ? Number(options.timeout) * 1000 : null,
|
|
1247
|
-
signal: interrupt.signal,
|
|
1248
|
-
onProgress: (line) => info(line),
|
|
1249
|
-
});
|
|
1250
|
-
}
|
|
1251
|
-
finally {
|
|
1252
|
-
process.off("SIGINT", onSignal);
|
|
1253
|
-
}
|
|
1254
|
-
if (outcome.message)
|
|
1255
|
-
error(outcome.message);
|
|
1256
|
-
// #3598 — the two sides, side by side. The session's own `timings`
|
|
1257
|
-
// say where the platform's share went; `client` says what this
|
|
1258
|
-
// process spent before and around it, so a slow load can be blamed on
|
|
1259
|
-
// the side that was actually slow.
|
|
1260
|
-
const clientTimings = { buildMs, ...outcome.timings };
|
|
1261
|
-
if (options.json) {
|
|
1262
|
-
json({
|
|
1263
|
-
...(outcome.session ?? { sessionId: outcome.sessionId }),
|
|
1264
|
-
client: clientTimings,
|
|
1265
|
-
});
|
|
1266
|
-
}
|
|
1267
|
-
else {
|
|
1268
|
-
if (outcome.session) {
|
|
1269
|
-
printIngestSession(outcome.session);
|
|
1270
|
-
}
|
|
1271
|
-
else if (outcome.sessionId) {
|
|
1272
|
-
printResult("Session", outcome.sessionId);
|
|
1273
|
-
}
|
|
1274
|
-
for (const [label, value] of ingestClientRows(clientTimings)) {
|
|
1275
|
-
printResult(label, value);
|
|
1276
|
-
}
|
|
1277
|
-
}
|
|
1278
|
-
return outcome.exitCode;
|
|
1279
|
-
};
|
|
1280
|
-
let code = 1;
|
|
1281
|
-
try {
|
|
1282
|
-
code = await run();
|
|
1283
|
-
}
|
|
1284
|
-
catch (err) {
|
|
1285
|
-
error(err.message);
|
|
1286
|
-
code = 1;
|
|
1287
|
-
}
|
|
1288
|
-
finally {
|
|
1289
|
-
await plan?.cleanup().catch(() => undefined);
|
|
1290
|
-
}
|
|
1291
|
-
// The result has to reach the pipe before the process ends: stdout to a
|
|
1292
|
-
// pipe is asynchronous and `process.exit` does not wait for it.
|
|
1293
|
-
await flushOutput();
|
|
1294
|
-
process.exit(code);
|
|
1024
|
+
await ingestAction(DOCUMENT_LOAD_ROUTES, documentId, options);
|
|
1295
1025
|
});
|
|
1296
1026
|
// Bulk-load sessions of a large document (#3434, criterion 9): a data noun
|
|
1297
1027
|
// with `list` and `get` readers, per docs/cli-design.md, beside `snapshots`
|
|
1298
1028
|
// below. The ACTION verb `documents ingest` is #3435's; these two are how an
|
|
1299
|
-
// operator sees what a session did and why one stopped
|
|
1029
|
+
// operator sees what a session did and why one stopped, and `abort` (#4475)
|
|
1030
|
+
// is how they give one up.
|
|
1300
1031
|
const ingests = documents
|
|
1301
1032
|
.command("ingests")
|
|
1302
|
-
.description("Inspect a large document's bulk-load sessions");
|
|
1033
|
+
.description("Inspect or abort a large document's bulk-load sessions");
|
|
1303
1034
|
ingests
|
|
1304
1035
|
.command("list")
|
|
1305
1036
|
.description("List a large document's bulk-load sessions, newest first")
|
|
@@ -1309,42 +1040,7 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1309
1040
|
.option("--cursor <cursor>", "Continue from a previous page's nextCursor")
|
|
1310
1041
|
.option("--json", "Output as JSON")
|
|
1311
1042
|
.action(async (documentId, options) => {
|
|
1312
|
-
|
|
1313
|
-
const resolvedAppId = resolveAppId(undefined, options);
|
|
1314
|
-
const client = new ApiClient();
|
|
1315
|
-
try {
|
|
1316
|
-
const page = normalizeCliListEnvelope(await client.listDocumentIngests(resolvedAppId, documentId, {
|
|
1317
|
-
limit,
|
|
1318
|
-
...(options.cursor ? { cursor: options.cursor } : {}),
|
|
1319
|
-
}));
|
|
1320
|
-
if (options.json) {
|
|
1321
|
-
json(page);
|
|
1322
|
-
return;
|
|
1323
|
-
}
|
|
1324
|
-
if (page.items.length === 0) {
|
|
1325
|
-
info("No ingest sessions yet.");
|
|
1326
|
-
return;
|
|
1327
|
-
}
|
|
1328
|
-
console.log(formatTable(page.items, [
|
|
1329
|
-
{ header: "SESSION", key: "sessionId" },
|
|
1330
|
-
{ header: "STATE", key: "state" },
|
|
1331
|
-
{ header: "CREATED", key: "createdAt", format: formatDate },
|
|
1332
|
-
{ header: "CHUNKS", key: "chunks" },
|
|
1333
|
-
{ header: "ROWS", key: "rows" },
|
|
1334
|
-
{
|
|
1335
|
-
header: "COMPLETED",
|
|
1336
|
-
key: "completedAt",
|
|
1337
|
-
format: (at) => (at ? formatDate(at) : "—"),
|
|
1338
|
-
},
|
|
1339
|
-
]));
|
|
1340
|
-
if (page.hasMore && page.nextCursor) {
|
|
1341
|
-
info(`More sessions: --cursor ${page.nextCursor}`);
|
|
1342
|
-
}
|
|
1343
|
-
}
|
|
1344
|
-
catch (err) {
|
|
1345
|
-
error(err.message);
|
|
1346
|
-
process.exit(1);
|
|
1347
|
-
}
|
|
1043
|
+
await listIngestsAction(DOCUMENT_LOAD_ROUTES, documentId, options);
|
|
1348
1044
|
});
|
|
1349
1045
|
ingests
|
|
1350
1046
|
.command("get")
|
|
@@ -1354,27 +1050,28 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1354
1050
|
.option("--app <app-id>", "App ID")
|
|
1355
1051
|
.option("--json", "Output as JSON")
|
|
1356
1052
|
.action(async (documentId, sessionId, options) => {
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1053
|
+
await getIngestAction(DOCUMENT_LOAD_ROUTES, documentId, sessionId, options);
|
|
1054
|
+
});
|
|
1055
|
+
// Destructive — what the session uploaded is discarded — so it confirms
|
|
1056
|
+
// unless `-y`, and a non-TTY stdin without `-y` is refused before any
|
|
1057
|
+
// request, as `documents ingest` refuses it (#4475).
|
|
1058
|
+
ingests
|
|
1059
|
+
.command("abort")
|
|
1060
|
+
.description("Abort a bulk-load session and discard what it uploaded")
|
|
1061
|
+
.argument("<document-id>", "Document ID")
|
|
1062
|
+
.argument("<session-id>", "Session ID")
|
|
1063
|
+
.option("--app <app-id>", "App ID")
|
|
1064
|
+
.option("-y, --yes", "Skip the confirmation prompt")
|
|
1065
|
+
.option("--json", "Output as JSON")
|
|
1066
|
+
.action(async (documentId, sessionId, options) => {
|
|
1067
|
+
await abortIngestAction(DOCUMENT_LOAD_ROUTES, documentId, sessionId, options);
|
|
1371
1068
|
});
|
|
1372
1069
|
// Snapshot builds of a large document (#3432, criterion 9): a data noun
|
|
1373
1070
|
// with `list` and `get` readers, per docs/cli-design.md, over the app API's
|
|
1374
1071
|
// read-only inspection routes.
|
|
1375
1072
|
const snapshots = documents
|
|
1376
1073
|
.command("snapshots")
|
|
1377
|
-
.description("Inspect a large document's
|
|
1074
|
+
.description("Inspect a large document's snapshots (its segments)");
|
|
1378
1075
|
snapshots
|
|
1379
1076
|
.command("list")
|
|
1380
1077
|
.description("List a large document's snapshot builds, newest first")
|
|
@@ -1384,77 +1081,17 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1384
1081
|
.option("--cursor <cursor>", "Continue from a previous page's nextCursor")
|
|
1385
1082
|
.option("--json", "Output as JSON")
|
|
1386
1083
|
.action(async (documentId, options) => {
|
|
1387
|
-
|
|
1388
|
-
const resolvedAppId = resolveAppId(undefined, options);
|
|
1389
|
-
const client = new ApiClient();
|
|
1390
|
-
try {
|
|
1391
|
-
const page = normalizeCliListEnvelope(await client.listDocumentSnapshots(resolvedAppId, documentId, {
|
|
1392
|
-
limit,
|
|
1393
|
-
...(options.cursor ? { cursor: options.cursor } : {}),
|
|
1394
|
-
}));
|
|
1395
|
-
if (options.json) {
|
|
1396
|
-
json(page);
|
|
1397
|
-
return;
|
|
1398
|
-
}
|
|
1399
|
-
if (page.items.length === 0) {
|
|
1400
|
-
info("No snapshot builds yet.");
|
|
1401
|
-
return;
|
|
1402
|
-
}
|
|
1403
|
-
console.log(formatTable(page.items, [
|
|
1404
|
-
{ header: "BUILD", key: "buildId" },
|
|
1405
|
-
{ header: "EPOCH", key: "epoch" },
|
|
1406
|
-
{ header: "STATE", key: "state" },
|
|
1407
|
-
{ header: "SOURCE", key: "source" },
|
|
1408
|
-
{
|
|
1409
|
-
header: "VERIFICATION",
|
|
1410
|
-
key: "verification",
|
|
1411
|
-
format: (verification) => verification
|
|
1412
|
-
? verification.code
|
|
1413
|
-
? `${verification.state} (${verification.code})`
|
|
1414
|
-
: verification.state
|
|
1415
|
-
: "-",
|
|
1416
|
-
},
|
|
1417
|
-
{ header: "ROWS", key: "rows" },
|
|
1418
|
-
{ header: "CHUNKS", key: "chunks" },
|
|
1419
|
-
{ header: "STARTED", key: "startedAt", format: formatDate },
|
|
1420
|
-
{ header: "COMPLETED", key: "completedAt", format: formatDate },
|
|
1421
|
-
]));
|
|
1422
|
-
if (page.hasMore && page.nextCursor) {
|
|
1423
|
-
info(`More builds: --cursor ${page.nextCursor}`);
|
|
1424
|
-
}
|
|
1425
|
-
}
|
|
1426
|
-
catch (err) {
|
|
1427
|
-
error(err.message);
|
|
1428
|
-
process.exit(1);
|
|
1429
|
-
}
|
|
1084
|
+
await listSnapshotsAction(DOCUMENT_LOAD_ROUTES, documentId, options);
|
|
1430
1085
|
});
|
|
1431
1086
|
snapshots
|
|
1432
1087
|
.command("get")
|
|
1433
|
-
.description("Show one snapshot
|
|
1088
|
+
.description("Show one snapshot (a segment): its state, rows and chunks")
|
|
1434
1089
|
.argument("<document-id>", "Document ID")
|
|
1435
1090
|
.argument("<build-id>", "Build ID")
|
|
1436
1091
|
.option("--app <app-id>", "App ID")
|
|
1437
1092
|
.option("--json", "Output as JSON")
|
|
1438
1093
|
.action(async (documentId, buildId, options) => {
|
|
1439
|
-
|
|
1440
|
-
const client = new ApiClient();
|
|
1441
|
-
try {
|
|
1442
|
-
const build = await client.getDocumentSnapshot(resolvedAppId, documentId, buildId);
|
|
1443
|
-
if (options.json) {
|
|
1444
|
-
json(build);
|
|
1445
|
-
return;
|
|
1446
|
-
}
|
|
1447
|
-
// The rows are composed by a pure function (#3435), so what a
|
|
1448
|
-
// terminal shows is checkable without spawning one; the command still
|
|
1449
|
-
// owns the printing.
|
|
1450
|
-
for (const [label, value] of snapshotBuildRows(build)) {
|
|
1451
|
-
printResult(label, value);
|
|
1452
|
-
}
|
|
1453
|
-
}
|
|
1454
|
-
catch (err) {
|
|
1455
|
-
error(err.message);
|
|
1456
|
-
process.exit(1);
|
|
1457
|
-
}
|
|
1094
|
+
await getSnapshotAction(DOCUMENT_LOAD_ROUTES, documentId, buildId, options);
|
|
1458
1095
|
});
|
|
1459
1096
|
// #3666, criterion 7 — the one ACTION verb in this group: snapshot the
|
|
1460
1097
|
// document now instead of waiting for the size or age trigger. Its exit
|
|
@@ -1467,152 +1104,14 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1467
1104
|
// on its own several times a day.
|
|
1468
1105
|
snapshots
|
|
1469
1106
|
.command("build")
|
|
1470
|
-
.description("Snapshot a large document now:
|
|
1107
|
+
.description("Snapshot a large document now: start a segment export of its records")
|
|
1471
1108
|
.argument("<document-id>", "Document ID")
|
|
1472
1109
|
.option("--app <app-id>", "App ID")
|
|
1473
|
-
.option("--wait", "Poll until the
|
|
1110
|
+
.option("--wait", "Poll until the segment is complete or failed")
|
|
1474
1111
|
.option("--timeout <seconds>", "Give up waiting after this many seconds")
|
|
1475
1112
|
.option("--json", "Output as JSON")
|
|
1476
1113
|
.action(async (documentId, options) => {
|
|
1477
|
-
|
|
1478
|
-
const client = new ApiClient();
|
|
1479
|
-
const run = async () => {
|
|
1480
|
-
// Ctrl-C while polling leaves the build running — by then the room is
|
|
1481
|
-
// already building it and giving up on watching is not giving up on
|
|
1482
|
-
// the build — so the signal is handed to the driver rather than
|
|
1483
|
-
// killing the process.
|
|
1484
|
-
const interrupt = new AbortController();
|
|
1485
|
-
const onSignal = () => interrupt.abort();
|
|
1486
|
-
process.on("SIGINT", onSignal);
|
|
1487
|
-
let outcome;
|
|
1488
|
-
try {
|
|
1489
|
-
outcome = await driveSnapshotBuild({
|
|
1490
|
-
client: {
|
|
1491
|
-
requestSnapshot: (id) => client.requestDocumentSnapshot(resolvedAppId, id),
|
|
1492
|
-
getSnapshot: (id, buildId) => client.getDocumentSnapshot(resolvedAppId, id, buildId),
|
|
1493
|
-
},
|
|
1494
|
-
documentId,
|
|
1495
|
-
wait: Boolean(options.wait),
|
|
1496
|
-
timeoutMs: options.timeout ? Number(options.timeout) * 1000 : null,
|
|
1497
|
-
signal: interrupt.signal,
|
|
1498
|
-
onProgress: (line) => info(line),
|
|
1499
|
-
});
|
|
1500
|
-
}
|
|
1501
|
-
finally {
|
|
1502
|
-
process.off("SIGINT", onSignal);
|
|
1503
|
-
}
|
|
1504
|
-
if (outcome.message)
|
|
1505
|
-
error(outcome.message);
|
|
1506
|
-
if (options.json) {
|
|
1507
|
-
// The route's answer as received, with the build the wait settled
|
|
1508
|
-
// on beside it when there was one — so a script branches on the
|
|
1509
|
-
// verdict rather than on an exit code alone.
|
|
1510
|
-
json(outcome.build ? { ...outcome.response, build: outcome.build } : outcome.response);
|
|
1511
|
-
return outcome.exitCode;
|
|
1512
|
-
}
|
|
1513
|
-
if (outcome.response?.sealed === false) {
|
|
1514
|
-
printResult("Sealed", "no — the open epoch carried nothing");
|
|
1515
|
-
printResult("Reason", outcome.response.reason);
|
|
1516
|
-
printResult("Covering build", outcome.response.coveringBuildId ?? "—");
|
|
1517
|
-
return outcome.exitCode;
|
|
1518
|
-
}
|
|
1519
|
-
printResult("Sealed epoch", outcome.response.sealedEpoch);
|
|
1520
|
-
printResult("Open epoch", outcome.response.nextEpoch);
|
|
1521
|
-
printResult("Build", outcome.response.buildId);
|
|
1522
|
-
if (outcome.build) {
|
|
1523
|
-
for (const [label, value] of snapshotBuildRows(outcome.build)) {
|
|
1524
|
-
printResult(label, value);
|
|
1525
|
-
}
|
|
1526
|
-
}
|
|
1527
|
-
return outcome.exitCode;
|
|
1528
|
-
};
|
|
1529
|
-
let code = 1;
|
|
1530
|
-
try {
|
|
1531
|
-
code = await run();
|
|
1532
|
-
}
|
|
1533
|
-
catch (err) {
|
|
1534
|
-
// The server's own sentence, plus the code a script branches on
|
|
1535
|
-
// (#3403) — a refused loop that only says "too young to seal" leaves
|
|
1536
|
-
// a caller parsing prose to tell it from a permission refusal.
|
|
1537
|
-
error(err.code ? `${err.message} (${err.code})` : err.message);
|
|
1538
|
-
const retryAfterMs = err.details?.retryAfterMs;
|
|
1539
|
-
if (typeof retryAfterMs === "number") {
|
|
1540
|
-
error(`Ask again in ${Math.ceil(retryAfterMs / 1000)}s.`);
|
|
1541
|
-
}
|
|
1542
|
-
code = 1;
|
|
1543
|
-
}
|
|
1544
|
-
// The result has to reach the pipe before the process ends: stdout to a
|
|
1545
|
-
// pipe is asynchronous and `process.exit` does not wait for it.
|
|
1546
|
-
await flushOutput();
|
|
1547
|
-
process.exit(code);
|
|
1548
|
-
});
|
|
1549
|
-
// #3433, criterion 10 — the operator audit: `snapshot ⊕ overlays == table`,
|
|
1550
|
-
// recomputed on this side with the CLIENT's loader and fold and compared
|
|
1551
|
-
// with the authoritative table page by page. The one check that can catch a
|
|
1552
|
-
// DO/client fold divergence, which per-build verification runs on one side
|
|
1553
|
-
// of and therefore cannot see.
|
|
1554
|
-
snapshots
|
|
1555
|
-
.command("audit")
|
|
1556
|
-
.description("Recompute a large document's snapshot chain locally and compare it with the table")
|
|
1557
|
-
.argument("<document-id>", "Document ID")
|
|
1558
|
-
.option("--app <app-id>", "App ID")
|
|
1559
|
-
.option("--retries <n>", "Attempts before giving up on a document that keeps changing", "3")
|
|
1560
|
-
.option("--json", "Output as JSON")
|
|
1561
|
-
.action(async (documentId, options) => {
|
|
1562
|
-
const resolvedAppId = resolveAppId(undefined, options);
|
|
1563
|
-
const client = new ApiClient();
|
|
1564
|
-
const report = await auditDocumentChain(client, resolvedAppId, documentId, Number(options.retries));
|
|
1565
|
-
if (options.json) {
|
|
1566
|
-
json(report);
|
|
1567
|
-
}
|
|
1568
|
-
else if (report.verdict === "none") {
|
|
1569
|
-
// No verdict is not a clean bill of health, so nothing that could be
|
|
1570
|
-
// read as one is printed: the reason, and nothing else.
|
|
1571
|
-
error(`No verdict: ${report.reason}`);
|
|
1572
|
-
}
|
|
1573
|
-
else {
|
|
1574
|
-
printResult("Document", documentId);
|
|
1575
|
-
printResult("Base build", report.buildId);
|
|
1576
|
-
printResult("Base epoch", report.epoch);
|
|
1577
|
-
printResult("Attempts", report.attempts);
|
|
1578
|
-
printResult("Verdict", report.verdict === "match"
|
|
1579
|
-
? "match — the table is the chain"
|
|
1580
|
-
: "DIVERGENCE");
|
|
1581
|
-
console.log(formatTable(report.models, [
|
|
1582
|
-
{ header: "MODEL", key: "model" },
|
|
1583
|
-
{ header: "COMPARED", key: "compared" },
|
|
1584
|
-
{
|
|
1585
|
-
header: "ONLY IN CHAIN",
|
|
1586
|
-
key: "onlyRecomputed",
|
|
1587
|
-
format: (ids) => (ids.length ? ids.join(", ") : "-"),
|
|
1588
|
-
},
|
|
1589
|
-
{
|
|
1590
|
-
header: "ONLY IN TABLE",
|
|
1591
|
-
key: "onlyTable",
|
|
1592
|
-
format: (ids) => (ids.length ? ids.join(", ") : "-"),
|
|
1593
|
-
},
|
|
1594
|
-
{
|
|
1595
|
-
header: "DIFFERING",
|
|
1596
|
-
key: "differing",
|
|
1597
|
-
format: (rows) => rows.length
|
|
1598
|
-
? rows.map((row) => `${row.id} (${row.fields.join(", ")})`).join("; ")
|
|
1599
|
-
: "-",
|
|
1600
|
-
},
|
|
1601
|
-
]));
|
|
1602
|
-
if (report.models.some((model) => model.truncated)) {
|
|
1603
|
-
info("Some lists were truncated; re-run with --json for the full report.");
|
|
1604
|
-
}
|
|
1605
|
-
if (report.verdict === "match")
|
|
1606
|
-
success("snapshot ⊕ overlays == table");
|
|
1607
|
-
}
|
|
1608
|
-
// The report has to reach the pipe before the process ends: stdout to a
|
|
1609
|
-
// pipe is asynchronous and `process.exit` does not wait for it, so a
|
|
1610
|
-
// large report piped into anything that is not reading as fast as this
|
|
1611
|
-
// writes used to arrive cut off mid-document (3433-C12).
|
|
1612
|
-
await flushOutput();
|
|
1613
|
-
// 0 match, 1 divergence, 2 no verdict — so the validation child can
|
|
1614
|
-
// script on "no verdict" without reading the text.
|
|
1615
|
-
process.exit(report.verdict === "match" ? 0 : report.verdict === "divergence" ? 1 : 2);
|
|
1114
|
+
await buildSnapshotAction(DOCUMENT_LOAD_ROUTES, documentId, options);
|
|
1616
1115
|
});
|
|
1617
1116
|
// Transfer document ownership
|
|
1618
1117
|
documents
|
|
@@ -1907,7 +1406,7 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1907
1406
|
await exportSingleDocument(client, resolvedAppId, documentId, options.output, options.blobs !== false ? true : false, options.json);
|
|
1908
1407
|
}
|
|
1909
1408
|
catch (err) {
|
|
1910
|
-
error(err
|
|
1409
|
+
error(describeDocumentFailure(err));
|
|
1911
1410
|
process.exit(1);
|
|
1912
1411
|
}
|
|
1913
1412
|
});
|
|
@@ -1925,6 +1424,9 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1925
1424
|
const resolvedAppId = resolveAppId(appIdArg, options);
|
|
1926
1425
|
const client = new ApiClient();
|
|
1927
1426
|
const includeBlobs = options.blobs !== false;
|
|
1427
|
+
// A document that fails does not stop the others; the run reports each
|
|
1428
|
+
// one by id and exits non-zero (#4149).
|
|
1429
|
+
const failed = [];
|
|
1928
1430
|
try {
|
|
1929
1431
|
const documents = await client.listAdminDocuments(resolvedAppId, options.userId);
|
|
1930
1432
|
let docList = Array.isArray(documents) ? documents : [];
|
|
@@ -1945,6 +1447,7 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1945
1447
|
}
|
|
1946
1448
|
catch (err) {
|
|
1947
1449
|
warn(`Failed to export ${docId}: ${err.message}`);
|
|
1450
|
+
failed.push({ documentId: docId, error: String(err?.message ?? err) });
|
|
1948
1451
|
}
|
|
1949
1452
|
}
|
|
1950
1453
|
// Write manifest. `export-all` is per-user, and each run REPLACES the
|
|
@@ -1979,16 +1482,25 @@ Permissions (enforced by the server, not the CLI):
|
|
|
1979
1482
|
fs.mkdirSync(path.dirname(manifestPath), { recursive: true });
|
|
1980
1483
|
fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
|
|
1981
1484
|
if (options.json) {
|
|
1982
|
-
json(manifest);
|
|
1485
|
+
json(failed.length > 0 ? { ...manifest, failed } : manifest);
|
|
1983
1486
|
}
|
|
1984
1487
|
else {
|
|
1985
1488
|
success(`Exported ${exportedIds.length} document(s) to ${options.output}`);
|
|
1489
|
+
if (failed.length > 0) {
|
|
1490
|
+
error(`Failed to export ${failed.length} document(s): ` +
|
|
1491
|
+
`${failed.map((entry) => entry.documentId).join(", ")}. ` +
|
|
1492
|
+
`The manifest lists only the documents exported.`);
|
|
1493
|
+
}
|
|
1986
1494
|
}
|
|
1987
1495
|
}
|
|
1988
1496
|
catch (err) {
|
|
1989
1497
|
error(err.message);
|
|
1990
1498
|
process.exit(1);
|
|
1991
1499
|
}
|
|
1500
|
+
if (failed.length > 0) {
|
|
1501
|
+
await flushOutput();
|
|
1502
|
+
process.exit(1);
|
|
1503
|
+
}
|
|
1992
1504
|
});
|
|
1993
1505
|
documents
|
|
1994
1506
|
.command("import")
|
|
@@ -2001,9 +1513,8 @@ Permissions (enforced by the server, not the CLI):
|
|
|
2001
1513
|
// `--overwrite` has never replaced a document: the import path issues no
|
|
2002
1514
|
// delete and calls `import/state`, whose room handler merges the exported
|
|
2003
1515
|
// state into the document with `Y.applyUpdate`. The description says so
|
|
2004
|
-
// (#3135), and names the one id remapping on top of it
|
|
2005
|
-
|
|
2006
|
-
.option("--overwrite", "Merge the exported state into a document that already exists (a root document export is merged into the target user's root document; a large-document export is installed, so it needs a document that holds nothing yet)")
|
|
1516
|
+
// (#3135), and names the one id remapping on top of it.
|
|
1517
|
+
.option("--overwrite", "Merge the exported state into a document that already exists (a root document export is merged into the target user's root document)")
|
|
2007
1518
|
.option("--dry-run", "Show what would be imported without making changes")
|
|
2008
1519
|
.option("--owner <userId-or-email>", "User ID or email to set as document owner (admin only)")
|
|
2009
1520
|
.option("--json", "Output as JSON")
|
|
@@ -2023,6 +1534,7 @@ Permissions (enforced by the server, not the CLI):
|
|
|
2023
1534
|
process.exit(1);
|
|
2024
1535
|
}
|
|
2025
1536
|
const client = new ApiClient();
|
|
1537
|
+
let failedCount = 0;
|
|
2026
1538
|
// Resolve --owner to a userId through the one shared lookup (#2763).
|
|
2027
1539
|
let ownerUserId;
|
|
2028
1540
|
if (options.owner) {
|
|
@@ -2068,6 +1580,30 @@ Permissions (enforced by the server, not the CLI):
|
|
|
2068
1580
|
info("No documents to import.");
|
|
2069
1581
|
return;
|
|
2070
1582
|
}
|
|
1583
|
+
// A record-log (format 3) export is a document's rows at one log
|
|
1584
|
+
// number, not a state to install (#4310): it is loaded with a bulk
|
|
1585
|
+
// load. Refused before any request, naming the way in.
|
|
1586
|
+
// A large-document (format 2) export cannot be installed: the platform
|
|
1587
|
+
// no longer serves the format. Its snapshot row chunks are the way
|
|
1588
|
+
// across, loaded into a record-log document. Refused before any write.
|
|
1589
|
+
const largeDocumentExports = docDirs.filter((docDir) => isLargeDocumentExport(docDir));
|
|
1590
|
+
if (largeDocumentExports.length > 0) {
|
|
1591
|
+
for (const docDir of largeDocumentExports) {
|
|
1592
|
+
error(`${docDir}: this export is a large document (format 2), which the platform no longer serves; ` +
|
|
1593
|
+
`create a record-log document and run \`primitive documents ingest <document-id> --input ${docDir}\` to load its row chunks`);
|
|
1594
|
+
}
|
|
1595
|
+
await flushOutput();
|
|
1596
|
+
process.exit(1);
|
|
1597
|
+
}
|
|
1598
|
+
const recordLogExports = docDirs.filter((docDir) => Number(readExportMetadata(docDir)?.documentFormat) === 3);
|
|
1599
|
+
if (recordLogExports.length > 0) {
|
|
1600
|
+
for (const docDir of recordLogExports) {
|
|
1601
|
+
error(`${docDir} is a record-log (format 3) export; \`documents import\` does not install one. ` +
|
|
1602
|
+
`Load its rows into a format-3 document with \`documents ingest <document-id> --input ${docDir}\`.`);
|
|
1603
|
+
}
|
|
1604
|
+
await flushOutput();
|
|
1605
|
+
process.exit(1);
|
|
1606
|
+
}
|
|
2071
1607
|
const summary = {
|
|
2072
1608
|
created: 0,
|
|
2073
1609
|
updated: 0,
|
|
@@ -2075,39 +1611,52 @@ Permissions (enforced by the server, not the CLI):
|
|
|
2075
1611
|
blobsUploaded: 0,
|
|
2076
1612
|
aliasesSet: 0,
|
|
2077
1613
|
aliasesSkipped: 0,
|
|
1614
|
+
completed: 0,
|
|
2078
1615
|
rootDocuments: [],
|
|
1616
|
+
failed: [],
|
|
2079
1617
|
};
|
|
2080
1618
|
// Where every root-marked export in this run lands, decided from reads
|
|
2081
1619
|
// alone and before any write (#3135): a refusal, or two exports
|
|
2082
1620
|
// competing for one user's root document, is reported up front.
|
|
2083
1621
|
const rootPlans = await planRootImports(client, resolvedAppId, docDirs, options.owner, ownerUserId, options.overwrite || false);
|
|
2084
|
-
//
|
|
2085
|
-
//
|
|
2086
|
-
// app half-populated (#2816). A root export is never installed as a
|
|
2087
|
-
// chain — `planRootImports` has already refused a format-2 one — so it
|
|
2088
|
-
// is not vouched for here.
|
|
2089
|
-
for (const docDir of docDirs) {
|
|
2090
|
-
if (rootPlans.has(docDir))
|
|
2091
|
-
continue;
|
|
2092
|
-
if (!largeDocumentArtifactAt(docDir))
|
|
2093
|
-
continue;
|
|
2094
|
-
const metadataPath = path.join(docDir, "metadata.json");
|
|
2095
|
-
const documentId = fs.existsSync(metadataPath)
|
|
2096
|
-
? JSON.parse(fs.readFileSync(metadataPath, "utf-8")).documentId
|
|
2097
|
-
: path.basename(docDir);
|
|
2098
|
-
readChainArtifact(docDir, documentId);
|
|
2099
|
-
}
|
|
1622
|
+
// One document failing does not stop the run (#4148): it is recorded
|
|
1623
|
+
// by id, the rest are imported, and the run exits non-zero at the end.
|
|
2100
1624
|
for (const docDir of docDirs) {
|
|
2101
|
-
|
|
1625
|
+
const progress = {
|
|
1626
|
+
documentId: readExportMetadata(docDir)?.documentId || path.basename(docDir),
|
|
1627
|
+
exists: false,
|
|
1628
|
+
};
|
|
1629
|
+
try {
|
|
1630
|
+
await importSingleDocument(client, resolvedAppId, docDir, options.overwrite || false, options.aliases || "skip", options.dryRun || false, summary, progress, ownerUserId, rootPlans.get(docDir));
|
|
1631
|
+
}
|
|
1632
|
+
catch (err) {
|
|
1633
|
+
const failure = {
|
|
1634
|
+
documentId: progress.documentId,
|
|
1635
|
+
status: progress.exists ? "incomplete" : "failed",
|
|
1636
|
+
error: err?.message || String(err),
|
|
1637
|
+
};
|
|
1638
|
+
summary.failed.push(failure);
|
|
1639
|
+
error(`Document ${failure.documentId} ` +
|
|
1640
|
+
`${failure.status === "incomplete" ? "is incomplete" : "failed"}: ` +
|
|
1641
|
+
failure.error);
|
|
1642
|
+
}
|
|
2102
1643
|
}
|
|
1644
|
+
failedCount = summary.failed.length;
|
|
2103
1645
|
if (options.json) {
|
|
2104
1646
|
json(summary);
|
|
2105
1647
|
}
|
|
2106
1648
|
else {
|
|
2107
|
-
|
|
1649
|
+
if (failedCount === 0) {
|
|
1650
|
+
success(`Import complete:`);
|
|
1651
|
+
}
|
|
1652
|
+
else {
|
|
1653
|
+
error(`Import finished, ${failedCount} document(s) not imported:`);
|
|
1654
|
+
}
|
|
2108
1655
|
keyValue(" Documents created", String(summary.created));
|
|
2109
1656
|
if (summary.updated > 0)
|
|
2110
1657
|
keyValue(" Documents updated", String(summary.updated));
|
|
1658
|
+
if (summary.completed > 0)
|
|
1659
|
+
keyValue(" Documents completed", String(summary.completed));
|
|
2111
1660
|
if (summary.skipped > 0)
|
|
2112
1661
|
keyValue(" Documents skipped", String(summary.skipped));
|
|
2113
1662
|
if (summary.blobsUploaded > 0)
|
|
@@ -2116,12 +1665,25 @@ Permissions (enforced by the server, not the CLI):
|
|
|
2116
1665
|
keyValue(" Aliases set", String(summary.aliasesSet));
|
|
2117
1666
|
if (summary.aliasesSkipped > 0)
|
|
2118
1667
|
keyValue(" Aliases skipped", String(summary.aliasesSkipped));
|
|
1668
|
+
for (const failure of summary.failed) {
|
|
1669
|
+
keyValue(` ${failure.documentId}`, `${failure.status}: ${failure.error}`);
|
|
1670
|
+
}
|
|
1671
|
+
if (failedCount > 0) {
|
|
1672
|
+
info("Re-run the same import to finish them: a document a run left " +
|
|
1673
|
+
"incomplete is completed, and one already imported is skipped.");
|
|
1674
|
+
}
|
|
2119
1675
|
}
|
|
2120
1676
|
}
|
|
2121
1677
|
catch (err) {
|
|
2122
1678
|
error(err.message);
|
|
2123
1679
|
process.exit(1);
|
|
2124
1680
|
}
|
|
1681
|
+
if (failedCount > 0) {
|
|
1682
|
+
// The summary names every failed document, and can be larger than a
|
|
1683
|
+
// pipe's buffer: it has to reach the pipe before the process ends.
|
|
1684
|
+
await flushOutput();
|
|
1685
|
+
process.exit(1);
|
|
1686
|
+
}
|
|
2125
1687
|
});
|
|
2126
1688
|
// Revoke group permission
|
|
2127
1689
|
groupPermissions
|
|
@@ -2159,153 +1721,79 @@ Permissions (enforced by the server, not the CLI):
|
|
|
2159
1721
|
}
|
|
2160
1722
|
});
|
|
2161
1723
|
}
|
|
2162
|
-
// ============================================
|
|
2163
|
-
// Helper functions for export/import
|
|
2164
|
-
// ============================================
|
|
2165
|
-
/**
|
|
2166
|
-
* Write a large document's export chain into `docDir` (#2816, behavior 20).
|
|
2167
|
-
*
|
|
2168
|
-
* The layout mirrors the chain and is applied in that order on import:
|
|
2169
|
-
*
|
|
2170
|
-
* ```
|
|
2171
|
-
* chain.json what this artifact is, and the order to apply it in
|
|
2172
|
-
* snapshot/ manifest.json + {model}/{n}.ndjson.gz, when there is a base
|
|
2173
|
-
* epochs/{E}.yjs each sealed overlay after the base's epoch, oldest first
|
|
2174
|
-
* current.yjs the open epoch's overlay
|
|
2175
|
-
* ```
|
|
2176
|
-
*
|
|
2177
|
-
* The server refuses a chain it cannot vouch for, so nothing here has to guess
|
|
2178
|
-
* whether an artifact is complete; what this function must not do is write a
|
|
2179
|
-
* PARTIAL one, which is why `chain.json` — the file that says "this is a
|
|
2180
|
-
* complete v2 export" — is written last.
|
|
2181
|
-
*
|
|
2182
|
-
* Returns the bytes written, which is what `--json` reports as the document's
|
|
2183
|
-
* size: for a large document that is the artifact, not one Yjs update.
|
|
2184
|
-
*/
|
|
2185
|
-
async function exportLargeDocumentChain(client, appId, documentId, docDir) {
|
|
2186
|
-
const chain = await client.exportDocumentChain(appId, documentId);
|
|
2187
|
-
let bytes = 0;
|
|
2188
|
-
// The open epoch's overlay travels inline: the rotation threshold bounds it.
|
|
2189
|
-
const current = Buffer.from(chain.current.update, "base64");
|
|
2190
|
-
fs.writeFileSync(path.join(docDir, "current.yjs"), current);
|
|
2191
|
-
bytes += current.byteLength;
|
|
2192
|
-
const overlays = [];
|
|
2193
|
-
if (Array.isArray(chain.overlays) && chain.overlays.length > 0) {
|
|
2194
|
-
const epochsDir = path.join(docDir, "epochs");
|
|
2195
|
-
fs.mkdirSync(epochsDir, { recursive: true });
|
|
2196
|
-
for (const overlay of chain.overlays) {
|
|
2197
|
-
const archive = await client.downloadDocumentArtifact(appId, overlay.download.path);
|
|
2198
|
-
fs.writeFileSync(path.join(epochsDir, `${overlay.epoch}.yjs`), archive);
|
|
2199
|
-
bytes += archive.byteLength;
|
|
2200
|
-
overlays.push({ epoch: overlay.epoch, sealedAt: overlay.sealedAt ?? null });
|
|
2201
|
-
}
|
|
2202
|
-
}
|
|
2203
|
-
let base = null;
|
|
2204
|
-
if (chain.base) {
|
|
2205
|
-
const snapshotDir = path.join(docDir, "snapshot");
|
|
2206
|
-
fs.mkdirSync(snapshotDir, { recursive: true });
|
|
2207
|
-
const manifestBytes = await client.downloadDocumentArtifact(appId, chain.base.download.path);
|
|
2208
|
-
const downloaded = JSON.parse(manifestBytes.toString("utf-8"));
|
|
2209
|
-
// Addressed within the granted build by each entry's `path` — the
|
|
2210
|
-
// own-build `{model}/{n}` or the `{epoch}-{buildId}/{model}/{n}` of a
|
|
2211
|
-
// chunk reused from an earlier build — never by key (#3432). On disk the
|
|
2212
|
-
// chunks are renumbered per model into the own-build form and the stored
|
|
2213
|
-
// manifest says so, which is the one layout `documents import` reads.
|
|
2214
|
-
const { manifest, files } = renumberManifestForExport(downloaded);
|
|
2215
|
-
for (const file of files) {
|
|
2216
|
-
const body = await client.downloadDocumentArtifact(appId, `${chain.base.download.path}/${file.download}`);
|
|
2217
|
-
const target = path.join(snapshotDir, file.file);
|
|
2218
|
-
fs.mkdirSync(path.dirname(target), { recursive: true });
|
|
2219
|
-
fs.writeFileSync(target, body);
|
|
2220
|
-
bytes += body.byteLength;
|
|
2221
|
-
}
|
|
2222
|
-
fs.writeFileSync(path.join(snapshotDir, "manifest.json"), JSON.stringify(manifest, null, 2));
|
|
2223
|
-
base = {
|
|
2224
|
-
epoch: chain.base.epoch,
|
|
2225
|
-
buildId: chain.base.buildId,
|
|
2226
|
-
rows: chain.base.rows,
|
|
2227
|
-
};
|
|
2228
|
-
}
|
|
2229
|
-
// Last, and without the signatures: a grant is this run's read of the
|
|
2230
|
-
// document, and an artifact that outlives it must not carry one.
|
|
2231
|
-
fs.writeFileSync(path.join(docDir, "chain.json"), JSON.stringify({
|
|
2232
|
-
version: 2,
|
|
2233
|
-
documentFormat: 2,
|
|
2234
|
-
documentId,
|
|
2235
|
-
epoch: chain.epoch,
|
|
2236
|
-
base,
|
|
2237
|
-
overlays,
|
|
2238
|
-
current: { epoch: chain.current.epoch, file: "current.yjs" },
|
|
2239
|
-
}, null, 2));
|
|
2240
|
-
return bytes;
|
|
2241
|
-
}
|
|
2242
|
-
/**
|
|
2243
|
-
* Where a stored manifest entry's chunk is on disk: `snapshot/{model}/{n}`.
|
|
2244
|
-
*
|
|
2245
|
-
* An exported manifest is in the own-build form (renumbered per model), so
|
|
2246
|
-
* the entry's `path` — or, for an artifact written before the field existed,
|
|
2247
|
-
* its key — names the file directly.
|
|
2248
|
-
*/
|
|
2249
|
-
function chunkFileOf(chunk) {
|
|
2250
|
-
const address = chunkAddressOf(chunk);
|
|
2251
|
-
return { model: address.model, index: address.index };
|
|
2252
|
-
}
|
|
2253
1724
|
async function exportSingleDocument(client, appId, documentId, outputDir, includeBlobs, jsonOutput) {
|
|
2254
1725
|
const docDir = path.join(outputDir, "documents", documentId);
|
|
2255
1726
|
fs.mkdirSync(docDir, { recursive: true });
|
|
1727
|
+
// `metadata.json` is what marks the directory a finished export — import
|
|
1728
|
+
// skips one without it — and it is written last. A rerun into an earlier
|
|
1729
|
+
// export's directory drops it before overwriting anything, so a run that
|
|
1730
|
+
// fails part-way leaves no new state beside the old aliases and grants
|
|
1731
|
+
// looking complete (#4149).
|
|
1732
|
+
fs.rmSync(path.join(docDir, "metadata.json"), { force: true });
|
|
2256
1733
|
// 1. Fetch document metadata
|
|
2257
1734
|
const docMeta = await client.getDocument(appId, documentId);
|
|
2258
|
-
// 2. Export the document's state. A legacy document is one blob; a
|
|
2259
|
-
//
|
|
2260
|
-
//
|
|
2261
|
-
// exists to avoid.
|
|
1735
|
+
// 2. Export the document's state. A legacy document is one blob; a
|
|
1736
|
+
// record-log document is its rows at one log number. A large document
|
|
1737
|
+
// (format 2) is no longer served, and the server refuses its export.
|
|
2262
1738
|
let stateBytes;
|
|
2263
|
-
if (Number(docMeta?.documentFormat) ===
|
|
2264
|
-
|
|
1739
|
+
if (Number(docMeta?.documentFormat) === 3) {
|
|
1740
|
+
// A record-log document (#4310): its current segment, with the rows
|
|
1741
|
+
// written while it exported merged in, so the directory is the document
|
|
1742
|
+
// at one log number.
|
|
1743
|
+
const exported = await exportRecordLogDocument({
|
|
1744
|
+
client: {
|
|
1745
|
+
requestSegment: () => client.requestDocumentSnapshot(appId, documentId),
|
|
1746
|
+
getSegment: (segmentId) => client.getDocumentSnapshot(appId, documentId, segmentId),
|
|
1747
|
+
listSegments: () => client.listDocumentSnapshots(appId, documentId, { limit: 100 }),
|
|
1748
|
+
exportChain: () => client.exportDocumentChain(appId, documentId),
|
|
1749
|
+
download: (artifactPath) => client.downloadDocumentArtifact(appId, artifactPath),
|
|
1750
|
+
},
|
|
1751
|
+
documentId,
|
|
1752
|
+
docDir,
|
|
1753
|
+
});
|
|
1754
|
+
stateBytes = exported.bytes;
|
|
2265
1755
|
}
|
|
2266
1756
|
else {
|
|
2267
1757
|
const stateResult = await client.exportDocumentState(appId, documentId);
|
|
2268
1758
|
fs.writeFileSync(path.join(docDir, "document.yjs"), Buffer.from(stateResult.state, "base64"));
|
|
2269
1759
|
stateBytes = stateResult.byteLength;
|
|
2270
1760
|
}
|
|
2271
|
-
// 3.
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
//
|
|
2280
|
-
//
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
1761
|
+
// 3. Read the permissions, pending invitations (deferred grants), aliases
|
|
1762
|
+
// and blob list — all before `permissions.json` or `metadata.json` is
|
|
1763
|
+
// written. Only a documented "none" answer reads as empty (a 404, and an
|
|
1764
|
+
// older server's 403/404 on pending invitations); any other failure fails
|
|
1765
|
+
// the document, because an export missing them reports success and the
|
|
1766
|
+
// import restores less than the document had (#4149).
|
|
1767
|
+
//
|
|
1768
|
+
// A root document cannot be shared, and the server answers both sharing
|
|
1769
|
+
// reads on one with a 403 (`validateNonRootDocument`). It has no grants
|
|
1770
|
+
// to export, so those reads are skipped rather than failing the document.
|
|
1771
|
+
const isRoot = isRootDocumentExport(docMeta?.tags);
|
|
1772
|
+
const permResult = isRoot
|
|
1773
|
+
? null
|
|
1774
|
+
: await readExportLookup(documentId, "permissions", () => client.listDocumentPermissions(appId, documentId));
|
|
1775
|
+
const pendingResult = isRoot
|
|
1776
|
+
? null
|
|
1777
|
+
: await readExportLookup(documentId, "pending invitations", () => client.listDocumentPendingInvitations(appId, documentId), { absentStatuses: PENDING_INVITATIONS_ABSENT_STATUSES });
|
|
1778
|
+
const aliasResult = await readExportLookup(documentId, "aliases", () => client.listDocumentAliases(appId, documentId));
|
|
1779
|
+
const blobs = includeBlobs
|
|
1780
|
+
? await readExportBlobList(client, appId, documentId)
|
|
1781
|
+
: [];
|
|
1782
|
+
// 4. Write permissions
|
|
2288
1783
|
const permissionsExport = buildPermissionsExport(permResult, pendingResult);
|
|
2289
1784
|
fs.writeFileSync(path.join(docDir, "permissions.json"), JSON.stringify(permissionsExport, null, 2));
|
|
2290
|
-
// 5.
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
}
|
|
2297
|
-
catch {
|
|
2298
|
-
// May not have aliases
|
|
2299
|
-
}
|
|
2300
|
-
// 6. Write metadata. A large document says so here as well as in
|
|
2301
|
-
// `chain.json`: the format decides how the artifact has to be installed,
|
|
2302
|
-
// and an import that reads only `metadata.json` must not mistake one for
|
|
2303
|
-
// a legacy document (#2816).
|
|
1785
|
+
// 5. Aliases (user-scoped only)
|
|
1786
|
+
const aliases = (Array.isArray(aliasResult) ? aliasResult : aliasResult?.aliases || []).filter((a) => (a.scope || a.aliasScope) === "user");
|
|
1787
|
+
// 6. Write metadata. A large or record-log document says so here as well
|
|
1788
|
+
// as in `chain.json`: the format decides how the artifact has to be
|
|
1789
|
+
// installed, and an import that reads only `metadata.json` must not
|
|
1790
|
+
// mistake one for a legacy document (#2816, #4310).
|
|
2304
1791
|
const metadata = {
|
|
2305
1792
|
documentId: docMeta.documentId || documentId,
|
|
2306
1793
|
title: docMeta.title,
|
|
2307
1794
|
tags: docMeta.tags || [],
|
|
2308
1795
|
...(Number(docMeta?.documentFormat) === 2 ? { documentFormat: 2 } : {}),
|
|
1796
|
+
...(Number(docMeta?.documentFormat) === 3 ? { documentFormat: 3 } : {}),
|
|
2309
1797
|
createdAt: docMeta.createdAt,
|
|
2310
1798
|
createdBy: docMeta.createdByEmail || docMeta.createdBy,
|
|
2311
1799
|
aliases: aliases.map((a) => ({
|
|
@@ -2316,14 +1804,6 @@ async function exportSingleDocument(client, appId, documentId, outputDir, includ
|
|
|
2316
1804
|
fs.writeFileSync(path.join(docDir, "metadata.json"), JSON.stringify(metadata, null, 2));
|
|
2317
1805
|
// 7. Blobs
|
|
2318
1806
|
if (includeBlobs) {
|
|
2319
|
-
let blobs = [];
|
|
2320
|
-
try {
|
|
2321
|
-
const blobResult = await client.listDocumentBlobs(appId, documentId);
|
|
2322
|
-
blobs = Array.isArray(blobResult) ? blobResult : blobResult?.items || blobResult?.blobs || [];
|
|
2323
|
-
}
|
|
2324
|
-
catch {
|
|
2325
|
-
// May not have blobs
|
|
2326
|
-
}
|
|
2327
1807
|
if (blobs.length > 0) {
|
|
2328
1808
|
const blobsDir = path.join(docDir, "blobs");
|
|
2329
1809
|
fs.mkdirSync(blobsDir, { recursive: true });
|
|
@@ -2357,210 +1837,47 @@ async function exportSingleDocument(client, appId, documentId, outputDir, includ
|
|
|
2357
1837
|
}
|
|
2358
1838
|
}
|
|
2359
1839
|
/**
|
|
2360
|
-
*
|
|
2361
|
-
*
|
|
2362
|
-
* Either marker is enough: `chain.json` is what the chain export writes last,
|
|
2363
|
-
* and `metadata.json` carries the format for artifacts inspected on their own.
|
|
2364
|
-
*/
|
|
2365
|
-
function largeDocumentArtifactAt(docDir) {
|
|
2366
|
-
if (fs.existsSync(path.join(docDir, "chain.json")))
|
|
2367
|
-
return true;
|
|
2368
|
-
const metadataPath = path.join(docDir, "metadata.json");
|
|
2369
|
-
if (!fs.existsSync(metadataPath))
|
|
2370
|
-
return false;
|
|
2371
|
-
try {
|
|
2372
|
-
const metadata = JSON.parse(fs.readFileSync(metadataPath, "utf-8"));
|
|
2373
|
-
return Number(metadata?.documentFormat) === 2;
|
|
2374
|
-
}
|
|
2375
|
-
catch {
|
|
2376
|
-
return false;
|
|
2377
|
-
}
|
|
2378
|
-
}
|
|
2379
|
-
/**
|
|
2380
|
-
* Read (and vouch for) a large document's chain artifact.
|
|
2381
|
-
*
|
|
2382
|
-
* `chain.json` is what the export writes LAST, so its absence means the
|
|
2383
|
-
* artifact is partial — a directory recognizable as a large document with no
|
|
2384
|
-
* description of what to install. Importing that as a legacy document would
|
|
2385
|
-
* read a `document.yjs` that is not there, create an empty document, and
|
|
2386
|
-
* report success: total data loss from the operator's point of view, since the
|
|
2387
|
-
* export is the only copy they were told they had. Refusing is the honest
|
|
2388
|
-
* answer, and it happens before anything is created.
|
|
2389
|
-
*/
|
|
2390
|
-
function readChainArtifact(docDir, documentId) {
|
|
2391
|
-
const chainPath = path.join(docDir, "chain.json");
|
|
2392
|
-
if (!fs.existsSync(chainPath)) {
|
|
2393
|
-
throw new Error(`Document ${documentId} was exported as a large document (documentFormat 2), ` +
|
|
2394
|
-
`but ${chainPath} is missing, so this artifact is incomplete. A large document's ` +
|
|
2395
|
-
`data IS its chain — a base snapshot, the sealed overlays after it, and the open ` +
|
|
2396
|
-
`epoch — and importing this directory as a legacy document would create an EMPTY ` +
|
|
2397
|
-
`document. Nothing was imported.`);
|
|
2398
|
-
}
|
|
2399
|
-
let chain;
|
|
2400
|
-
try {
|
|
2401
|
-
chain = JSON.parse(fs.readFileSync(chainPath, "utf-8"));
|
|
2402
|
-
}
|
|
2403
|
-
catch (parseError) {
|
|
2404
|
-
throw new Error(`Document ${documentId}: ${chainPath} is not readable as a chain description ` +
|
|
2405
|
-
`(${parseError.message}), so nothing was imported.`);
|
|
2406
|
-
}
|
|
2407
|
-
if (!Number.isSafeInteger(Number(chain?.epoch))) {
|
|
2408
|
-
throw new Error(`Document ${documentId}: ${chainPath} names no open epoch, so nothing was imported.`);
|
|
2409
|
-
}
|
|
2410
|
-
const artifact = {
|
|
2411
|
-
epoch: Number(chain.epoch),
|
|
2412
|
-
base: chain.base
|
|
2413
|
-
? {
|
|
2414
|
-
epoch: Number(chain.base.epoch),
|
|
2415
|
-
buildId: String(chain.base.buildId),
|
|
2416
|
-
rows: Number(chain.base.rows ?? 0),
|
|
2417
|
-
}
|
|
2418
|
-
: null,
|
|
2419
|
-
overlays: (Array.isArray(chain.overlays) ? chain.overlays : []).map((overlay) => ({ epoch: Number(overlay.epoch) })),
|
|
2420
|
-
};
|
|
2421
|
-
verifyChainArtifact(docDir, documentId, artifact);
|
|
2422
|
-
return artifact;
|
|
2423
|
-
}
|
|
2424
|
-
/**
|
|
2425
|
-
* Check that the directory holds every file the chain names, and that the
|
|
2426
|
-
* chain is one.
|
|
1840
|
+
* Every uploaded blob of a document, read page by page (#4149).
|
|
2427
1841
|
*
|
|
2428
|
-
*
|
|
2429
|
-
*
|
|
2430
|
-
*
|
|
2431
|
-
*
|
|
2432
|
-
*
|
|
2433
|
-
*
|
|
2434
|
-
*
|
|
1842
|
+
* The list answers one page at a time (`{ items, hasMore, nextCursor }`), so
|
|
1843
|
+
* a document with more blobs than a page holds would otherwise export only
|
|
1844
|
+
* the first page and report success. Each page is its own lookup: a 404 on
|
|
1845
|
+
* the first means the document has no blobs, but a later page that fails —
|
|
1846
|
+
* a 404 included — fails the document, because the list is already known to
|
|
1847
|
+
* be longer. A page that says more follow without a new cursor to read them
|
|
1848
|
+
* by fails the document too: stopping there exports part of the list, and
|
|
1849
|
+
* rereading the same cursor never ends.
|
|
2435
1850
|
*/
|
|
2436
|
-
function
|
|
2437
|
-
const
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
.
|
|
2446
|
-
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
(
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
if (chain.base) {
|
|
2458
|
-
const manifestPath = path.join(docDir, "snapshot", "manifest.json");
|
|
2459
|
-
require(manifestPath, "its base snapshot's manifest");
|
|
2460
|
-
let manifest;
|
|
2461
|
-
try {
|
|
2462
|
-
manifest = JSON.parse(fs.readFileSync(manifestPath, "utf-8"));
|
|
2463
|
-
}
|
|
2464
|
-
catch (parseError) {
|
|
2465
|
-
refuse(`has a base snapshot manifest that is not readable ` +
|
|
2466
|
-
`(${parseError.message})`);
|
|
2467
|
-
}
|
|
2468
|
-
// Validated where it enters (#3432): the fields the server keys on — an
|
|
2469
|
-
// entry's ordinal, its path, its range — fail silently rather than loudly
|
|
2470
|
-
// if they are wrong, and the server refuses the same manifest later with
|
|
2471
|
-
// the same message.
|
|
2472
|
-
try {
|
|
2473
|
-
validateSnapshotManifest(manifest);
|
|
2474
|
-
}
|
|
2475
|
-
catch (invalid) {
|
|
2476
|
-
refuse(`has a base snapshot manifest that is not valid (${invalid.message})`);
|
|
2477
|
-
}
|
|
2478
|
-
for (const chunk of manifest.chunks ?? []) {
|
|
2479
|
-
const { model, index } = chunkFileOf(chunk);
|
|
2480
|
-
require(path.join(docDir, "snapshot", model, `${index}.ndjson.gz`), `chunk ${index} of ${model}`);
|
|
2481
|
-
}
|
|
2482
|
-
}
|
|
2483
|
-
for (const overlay of chain.overlays) {
|
|
2484
|
-
require(path.join(docDir, "epochs", `${overlay.epoch}.yjs`), `the archived overlay of epoch ${overlay.epoch}`);
|
|
2485
|
-
}
|
|
2486
|
-
require(path.join(docDir, "current.yjs"), "its open epoch's overlay");
|
|
1851
|
+
async function readExportBlobList(client, appId, documentId) {
|
|
1852
|
+
const blobs = [];
|
|
1853
|
+
const seen = new Set();
|
|
1854
|
+
let cursor;
|
|
1855
|
+
do {
|
|
1856
|
+
const pageCursor = cursor;
|
|
1857
|
+
const page = await readExportLookup(documentId, "blob list", () => client.listDocumentBlobs(appId, documentId, pageCursor ? { cursor: pageCursor } : undefined), pageCursor ? { absentStatuses: [] } : {});
|
|
1858
|
+
if (Array.isArray(page))
|
|
1859
|
+
return [...blobs, ...page];
|
|
1860
|
+
blobs.push(...(page?.items || page?.blobs || []));
|
|
1861
|
+
if (!page?.hasMore)
|
|
1862
|
+
break;
|
|
1863
|
+
cursor = page?.nextCursor || undefined;
|
|
1864
|
+
if (!cursor || seen.has(cursor)) {
|
|
1865
|
+
throw new Error(`Document ${documentId}: reading its blob list failed — a page said ` +
|
|
1866
|
+
`more follow but gave no new cursor to read them by. Its export ` +
|
|
1867
|
+
`would be incomplete, so it was not exported.`);
|
|
1868
|
+
}
|
|
1869
|
+
seen.add(cursor);
|
|
1870
|
+
} while (cursor);
|
|
1871
|
+
return blobs;
|
|
2487
1872
|
}
|
|
2488
|
-
/** How long an install is waited on before the CLI gives up on it. */
|
|
2489
|
-
const CHAIN_INSTALL_TIMEOUT_MS = 60 * 60 * 1000;
|
|
2490
|
-
/** How often the install's progress is asked for. */
|
|
2491
|
-
const CHAIN_INSTALL_POLL_MS = 1000;
|
|
2492
1873
|
/**
|
|
2493
|
-
*
|
|
2494
|
-
* (#
|
|
2495
|
-
*
|
|
2496
|
-
* Every artifact is uploaded byte-for-byte into the new document's own prefix
|
|
2497
|
-
* — the manifest's digests are over the STORED bytes, so re-encoding a chunk
|
|
2498
|
-
* here would fail its own verification on install — and then the server
|
|
2499
|
-
* streams them into the document's `records` table a tick at a time. The full
|
|
2500
|
-
* document is never built as a Y.Doc on either side, which is what makes
|
|
2501
|
-
* importing a multi-hundred-MB artifact possible at all; it is also the
|
|
2502
|
-
* efficient way to bulk-seed a new large document.
|
|
1874
|
+
* One idempotent write of an import, retried through a transient failure
|
|
1875
|
+
* (#4148): a blob PUT, a state merge, an artifact upload or an install read.
|
|
1876
|
+
* Each retry is said, so a slow run is not mistaken for a stuck one.
|
|
2503
1877
|
*/
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
// here has to walk the directory and guess what belongs to the build.
|
|
2508
|
-
if (chain.base) {
|
|
2509
|
-
const manifestPath = path.join(docDir, "snapshot", "manifest.json");
|
|
2510
|
-
const manifestBytes = fs.readFileSync(manifestPath);
|
|
2511
|
-
await client.uploadDocumentChainArtifact(appId, documentId, {
|
|
2512
|
-
kind: "manifest",
|
|
2513
|
-
epoch: chain.base.epoch,
|
|
2514
|
-
buildId: chain.base.buildId,
|
|
2515
|
-
}, manifestBytes);
|
|
2516
|
-
const manifest = JSON.parse(manifestBytes.toString("utf-8"));
|
|
2517
|
-
for (const chunk of manifest.chunks ?? []) {
|
|
2518
|
-
// Addressed by the entry's `path` (#3432); the server installs by it too.
|
|
2519
|
-
const { model, index } = chunkFileOf(chunk);
|
|
2520
|
-
await client.uploadDocumentChainArtifact(appId, documentId, {
|
|
2521
|
-
kind: "chunk",
|
|
2522
|
-
epoch: chain.base.epoch,
|
|
2523
|
-
buildId: chain.base.buildId,
|
|
2524
|
-
model,
|
|
2525
|
-
index,
|
|
2526
|
-
}, fs.readFileSync(path.join(docDir, "snapshot", model, `${index}.ndjson.gz`)));
|
|
2527
|
-
}
|
|
2528
|
-
}
|
|
2529
|
-
// The sealed overlays, oldest first, then the open epoch's — which travels
|
|
2530
|
-
// archived exactly like a sealed one, so the imported document's chain is
|
|
2531
|
-
// the chain that was exported.
|
|
2532
|
-
for (const overlay of chain.overlays) {
|
|
2533
|
-
await client.uploadDocumentChainArtifact(appId, documentId, { kind: "overlay", epoch: overlay.epoch }, fs.readFileSync(path.join(docDir, "epochs", `${overlay.epoch}.yjs`)));
|
|
2534
|
-
}
|
|
2535
|
-
await client.uploadDocumentChainArtifact(appId, documentId, { kind: "overlay", epoch: chain.epoch }, fs.readFileSync(path.join(docDir, "current.yjs")));
|
|
2536
|
-
await client.installDocumentChain(appId, documentId, {
|
|
2537
|
-
base: chain.base
|
|
2538
|
-
? { epoch: chain.base.epoch, buildId: chain.base.buildId }
|
|
2539
|
-
: null,
|
|
2540
|
-
overlays: chain.overlays.map((overlay) => overlay.epoch),
|
|
2541
|
-
currentEpoch: chain.epoch,
|
|
2542
|
-
});
|
|
2543
|
-
// The install runs on the document's alarm, a bounded tick at a time, so it
|
|
2544
|
-
// is waited on rather than awaited: reporting "imported" while the records
|
|
2545
|
-
// are still arriving is the failure this whole path exists to avoid.
|
|
2546
|
-
const deadline = Date.now() + CHAIN_INSTALL_TIMEOUT_MS;
|
|
2547
|
-
for (;;) {
|
|
2548
|
-
const status = await client.getDocumentChainInstall(appId, documentId);
|
|
2549
|
-
const install = status?.install;
|
|
2550
|
-
if (install?.state === "complete")
|
|
2551
|
-
return;
|
|
2552
|
-
if (install?.state === "failed") {
|
|
2553
|
-
throw new Error(`Document ${documentId}: installing its chain failed — ` +
|
|
2554
|
-
`${install.error || "no reason given"}. The document is incomplete ` +
|
|
2555
|
-
`and should be deleted before the import is retried.`);
|
|
2556
|
-
}
|
|
2557
|
-
if (Date.now() >= deadline) {
|
|
2558
|
-
throw new Error(`Document ${documentId}: its chain install has not finished after ` +
|
|
2559
|
-
`${Math.round(CHAIN_INSTALL_TIMEOUT_MS / 60000)} minutes. It continues ` +
|
|
2560
|
-
`on the server; check with 'primitive documents get ${documentId}'.`);
|
|
2561
|
-
}
|
|
2562
|
-
await new Promise((resolve) => setTimeout(resolve, CHAIN_INSTALL_POLL_MS));
|
|
2563
|
-
}
|
|
1878
|
+
function retryingImportStep(what, attempt) {
|
|
1879
|
+
return withTransientRetry(attempt, (failure, delayMs) => warn(`${what} failed (${failure?.message || "no reason given"}); ` +
|
|
1880
|
+
`retrying in ${delayMs / 1000}s`));
|
|
2564
1881
|
}
|
|
2565
1882
|
/** Mirrors `ROOT_DOCUMENT_TAG` in `src/config/constants.ts`. */
|
|
2566
1883
|
const ROOT_DOCUMENT_TAG = "__ROOT_TAG__";
|
|
@@ -2569,18 +1886,14 @@ function isRootDocumentExport(tags) {
|
|
|
2569
1886
|
return Array.isArray(tags) && tags.includes(ROOT_DOCUMENT_TAG);
|
|
2570
1887
|
}
|
|
2571
1888
|
/**
|
|
2572
|
-
*
|
|
2573
|
-
*
|
|
2574
|
-
* A format-2 export is a chain installed wholesale into a freshly created
|
|
2575
|
-
* format-2 document — a replace, not a merge, and not something that can be
|
|
2576
|
-
* done to a root document, which the server only ever mints as a legacy
|
|
2577
|
-
* document. Refusing says so rather than installing a chain somewhere it does
|
|
2578
|
-
* not belong.
|
|
1889
|
+
* Whether `docDir` holds a large document's (format 2) export: the
|
|
1890
|
+
* `chain.json` its export wrote last, or a `metadata.json` naming format 2.
|
|
2579
1891
|
*/
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
1892
|
+
function isLargeDocumentExport(docDir) {
|
|
1893
|
+
if (fs.existsSync(path.join(docDir, "chain.json")))
|
|
1894
|
+
return true;
|
|
1895
|
+
return Number(readExportMetadata(docDir)?.documentFormat) === 2;
|
|
1896
|
+
}
|
|
2584
1897
|
/** Read an export directory's `metadata.json`, or null when it has none. */
|
|
2585
1898
|
function readExportMetadata(docDir) {
|
|
2586
1899
|
const metadataPath = path.join(docDir, "metadata.json");
|
|
@@ -2610,6 +1923,24 @@ function readExportMetadata(docDir) {
|
|
|
2610
1923
|
*/
|
|
2611
1924
|
async function planRootImports(client, appId, docDirs, explicitOwner, explicitOwnerUserId, overwrite) {
|
|
2612
1925
|
const plans = new Map();
|
|
1926
|
+
// The app's `rootDocumentFormat`, read once and only when some target user
|
|
1927
|
+
// has no root yet (#4195). A failed read fails the run: a guessed format is
|
|
1928
|
+
// a confident create into the wrong kind of root.
|
|
1929
|
+
let appSetting;
|
|
1930
|
+
const readAppSetting = async (exportedDocumentId) => {
|
|
1931
|
+
if (!appSetting) {
|
|
1932
|
+
try {
|
|
1933
|
+
const app = await client.getApp(appId);
|
|
1934
|
+
appSetting = { value: app?.rootDocumentFormat ?? null };
|
|
1935
|
+
}
|
|
1936
|
+
catch (error) {
|
|
1937
|
+
throw new Error(`Document ${exportedDocumentId}: reading app ${appId}'s ` +
|
|
1938
|
+
`rootDocumentFormat failed — ${error?.message || "no reason given"}. ` +
|
|
1939
|
+
`Nothing was imported.`);
|
|
1940
|
+
}
|
|
1941
|
+
}
|
|
1942
|
+
return appSetting.value;
|
|
1943
|
+
};
|
|
2613
1944
|
for (const docDir of docDirs) {
|
|
2614
1945
|
const metadata = readExportMetadata(docDir);
|
|
2615
1946
|
if (!metadata || !isRootDocumentExport(metadata.tags))
|
|
@@ -2621,16 +1952,13 @@ async function planRootImports(client, appId, docDirs, explicitOwner, explicitOw
|
|
|
2621
1952
|
exportedDocumentId,
|
|
2622
1953
|
owner,
|
|
2623
1954
|
action: "refused",
|
|
1955
|
+
exportFormat: rootDocumentFormatOf(metadata.documentFormat),
|
|
2624
1956
|
};
|
|
2625
1957
|
const refuse = (reason) => {
|
|
2626
1958
|
plan.action = "refused";
|
|
2627
1959
|
plan.reason = reason;
|
|
2628
1960
|
plans.set(docDir, plan);
|
|
2629
1961
|
};
|
|
2630
|
-
if (Number(metadata.documentFormat) === 2) {
|
|
2631
|
-
refuse(ROOT_LARGE_DOCUMENT_REFUSAL);
|
|
2632
|
-
continue;
|
|
2633
|
-
}
|
|
2634
1962
|
if (!owner) {
|
|
2635
1963
|
refuse(`it is a root document export that records no owner, so there is no ` +
|
|
2636
1964
|
`user in ${appId} to restore it into. Pass --owner <userId-or-email>.`);
|
|
@@ -2678,13 +2006,33 @@ async function planRootImports(client, appId, docDirs, explicitOwner, explicitOw
|
|
|
2678
2006
|
refuse(unresolvableRootOwner(appId, owner));
|
|
2679
2007
|
continue;
|
|
2680
2008
|
}
|
|
2009
|
+
// The export's format against the root it would land in (#4195): the
|
|
2010
|
+
// existing root's, or the one the app's setting says the mint produces.
|
|
2681
2011
|
const rootDocId = assignment?.rootDocId || null;
|
|
2682
|
-
if (
|
|
2683
|
-
plan.
|
|
2012
|
+
if (rootDocId)
|
|
2013
|
+
plan.targetRootDocId = rootDocId;
|
|
2014
|
+
const decision = decideRootImport({
|
|
2015
|
+
exportFormat: plan.exportFormat,
|
|
2016
|
+
existingRootFormat: rootDocId
|
|
2017
|
+
? rootDocumentFormatOf(assignment.documentFormat)
|
|
2018
|
+
: null,
|
|
2019
|
+
appSetting: rootDocId ? null : await readAppSetting(exportedDocumentId),
|
|
2020
|
+
overwrite,
|
|
2021
|
+
owner,
|
|
2022
|
+
...(rootDocId ? { targetRootDocId: rootDocId } : {}),
|
|
2023
|
+
});
|
|
2024
|
+
plan.targetFormat = decision.targetFormat;
|
|
2025
|
+
if (decision.outcome === "refuse") {
|
|
2026
|
+
refuse(decision.reason);
|
|
2027
|
+
continue;
|
|
2028
|
+
}
|
|
2029
|
+
if (decision.outcome === "skip") {
|
|
2030
|
+
plan.action = "skip";
|
|
2031
|
+
plan.reason = decision.reason;
|
|
2684
2032
|
}
|
|
2685
2033
|
else {
|
|
2686
|
-
plan.
|
|
2687
|
-
plan.
|
|
2034
|
+
plan.action = rootDocId ? "apply" : "create";
|
|
2035
|
+
plan.mode = decision.outcome;
|
|
2688
2036
|
}
|
|
2689
2037
|
plans.set(docDir, plan);
|
|
2690
2038
|
}
|
|
@@ -2727,7 +2075,7 @@ function assertDistinctRootTargets(plans) {
|
|
|
2727
2075
|
}
|
|
2728
2076
|
}
|
|
2729
2077
|
/**
|
|
2730
|
-
*
|
|
2078
|
+
* The target app's copy of this document, or null when it holds none.
|
|
2731
2079
|
*
|
|
2732
2080
|
* Only an explicit 404 answers "no" (#3096). An expired login (401), a lost
|
|
2733
2081
|
* permission (403), a server fault or a transport failure says nothing about
|
|
@@ -2736,19 +2084,18 @@ function assertDistinctRootTargets(plans) {
|
|
|
2736
2084
|
* silently omits the overwrite the operator asked for. Those propagate, and
|
|
2737
2085
|
* `documents import` fails the run.
|
|
2738
2086
|
*/
|
|
2739
|
-
async function
|
|
2087
|
+
async function existingDocument(client, appId, documentId) {
|
|
2740
2088
|
try {
|
|
2741
|
-
await client.getDocument(appId, documentId);
|
|
2742
|
-
return true;
|
|
2089
|
+
return (await client.getDocument(appId, documentId)) ?? {};
|
|
2743
2090
|
}
|
|
2744
2091
|
catch (error) {
|
|
2745
2092
|
if (error?.statusCode === 404)
|
|
2746
|
-
return
|
|
2093
|
+
return null;
|
|
2747
2094
|
throw new Error(`Document ${documentId}: checking whether it already exists failed — ` +
|
|
2748
2095
|
`${error?.message || "no reason given"}. Nothing was imported for it.`);
|
|
2749
2096
|
}
|
|
2750
2097
|
}
|
|
2751
|
-
async function importSingleDocument(client, appId, docDir, overwrite, aliasMode, dryRun, summary, ownerUserId, rootPlan) {
|
|
2098
|
+
async function importSingleDocument(client, appId, docDir, overwrite, aliasMode, dryRun, summary, progress, ownerUserId, rootPlan) {
|
|
2752
2099
|
const metadataPath = path.join(docDir, "metadata.json");
|
|
2753
2100
|
if (!fs.existsSync(metadataPath)) {
|
|
2754
2101
|
warn(`Skipping ${docDir}: no metadata.json found`);
|
|
@@ -2760,37 +2107,46 @@ async function importSingleDocument(client, appId, docDir, overwrite, aliasMode,
|
|
|
2760
2107
|
// A root-marked export is never restored under its exported id: that id is
|
|
2761
2108
|
// the SOURCE user's root document. It goes to the target user's own root
|
|
2762
2109
|
// instead, per the plan made before this run wrote anything (#3135).
|
|
2763
|
-
//
|
|
2764
|
-
// Decided BEFORE the chain artifact below is read, because a root export is
|
|
2765
|
-
// never installed as a chain: a large-document root is refused by the plan,
|
|
2766
|
-
// and reading its chain here would fail the whole run over an artifact this
|
|
2767
|
-
// run has already decided not to touch.
|
|
2768
2110
|
if (isRootDocumentExport(metadata.tags)) {
|
|
2769
|
-
await importRootDocument(client, appId, docDir, metadata, rootPlan, overwrite, aliasMode, dryRun, summary);
|
|
2111
|
+
await importRootDocument(client, appId, docDir, metadata, rootPlan, overwrite, aliasMode, dryRun, summary, progress);
|
|
2770
2112
|
return;
|
|
2771
2113
|
}
|
|
2772
|
-
// A large document's data is the chain, not `document.yjs` (#2816), and it
|
|
2773
|
-
// is installed rather than replayed. Vouched for before anything is created.
|
|
2774
|
-
const isLargeDocument = largeDocumentArtifactAt(docDir);
|
|
2775
|
-
if (isLargeDocument)
|
|
2776
|
-
readChainArtifact(docDir, documentId);
|
|
2777
2114
|
// `--dry-run` decides from the same facts the real run does (#3096), so a
|
|
2778
2115
|
// preview never promises a skip that the run turns into a merge: it looks
|
|
2779
2116
|
// the document up as well, and only the writes below are withheld.
|
|
2780
|
-
const
|
|
2117
|
+
const existing = await existingDocument(client, appId, documentId);
|
|
2118
|
+
const exists = existing !== null;
|
|
2119
|
+
progress.exists = exists;
|
|
2781
2120
|
const wouldSkip = dryRun ? "[dry-run] Would skip" : "Skipping";
|
|
2782
2121
|
if (exists && !overwrite) {
|
|
2783
|
-
//
|
|
2784
|
-
//
|
|
2785
|
-
// that
|
|
2786
|
-
// is
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
:
|
|
2793
|
-
|
|
2122
|
+
// A document an earlier run created and did not finish is completed, not
|
|
2123
|
+
// skipped (#4148): what it lacks of the export is read from the server,
|
|
2124
|
+
// and only that is written. A document that lacks nothing, or that holds
|
|
2125
|
+
// content of its own, is skipped, so a re-run never merges into or
|
|
2126
|
+
// re-uploads to one already imported, nor writes to one it did not start.
|
|
2127
|
+
// With an owner named, the export's user aliases are among what it can
|
|
2128
|
+
// lack.
|
|
2129
|
+
const gaps = await findImportGaps(client, appId, docDir, documentId, ownerUserId
|
|
2130
|
+
? { userId: ownerUserId, aliasMode, exported: metadata.aliases }
|
|
2131
|
+
: undefined);
|
|
2132
|
+
if (gaps) {
|
|
2133
|
+
if (dryRun) {
|
|
2134
|
+
info(`[dry-run] Would complete document ${documentId} ` +
|
|
2135
|
+
`(${metadata.title || "untitled"}): it is missing ${describeImportGaps(gaps)}`);
|
|
2136
|
+
summary.completed++;
|
|
2137
|
+
return;
|
|
2138
|
+
}
|
|
2139
|
+
if (gaps.state)
|
|
2140
|
+
await importExportedState(client, appId, docDir, documentId);
|
|
2141
|
+
await uploadExportedBlobs(client, appId, documentId, docDir, gaps.blobs, summary);
|
|
2142
|
+
await restoreExportedAliases(client, appId, documentId, ownerUserId ? { ...metadata, aliases: gaps.aliases } : metadata, ownerUserId, aliasMode, summary);
|
|
2143
|
+
summary.completed++;
|
|
2144
|
+
info(`Completed document ${documentId} (${metadata.title || "untitled"}): ` +
|
|
2145
|
+
`imported ${describeImportGaps(gaps)}`);
|
|
2146
|
+
return;
|
|
2147
|
+
}
|
|
2148
|
+
warn(`${wouldSkip} ${documentId}: already exists (use --overwrite to merge ` +
|
|
2149
|
+
`the exported state into it)`);
|
|
2794
2150
|
summary.skipped++;
|
|
2795
2151
|
return;
|
|
2796
2152
|
}
|
|
@@ -2803,8 +2159,7 @@ async function importSingleDocument(client, appId, docDir, overwrite, aliasMode,
|
|
|
2803
2159
|
summary.created++;
|
|
2804
2160
|
return;
|
|
2805
2161
|
}
|
|
2806
|
-
// Create document if it doesn't exist.
|
|
2807
|
-
// never migrated, so a large document has to be created as one (#2816).
|
|
2162
|
+
// Create document if it doesn't exist.
|
|
2808
2163
|
//
|
|
2809
2164
|
// Tags go back exactly as `metadata.json` recorded them (#3096): an export
|
|
2810
2165
|
// is a faithful record of what the document is, and editing the list on the
|
|
@@ -2814,72 +2169,217 @@ async function importSingleDocument(client, appId, docDir, overwrite, aliasMode,
|
|
|
2814
2169
|
title: metadata.title || "Imported Document",
|
|
2815
2170
|
documentId,
|
|
2816
2171
|
tags: metadata.tags,
|
|
2817
|
-
...(isLargeDocument ? { documentFormat: 2 } : {}),
|
|
2818
2172
|
...(ownerUserId ? { createdBy: ownerUserId } : {}),
|
|
2819
2173
|
});
|
|
2174
|
+
progress.exists = true;
|
|
2820
2175
|
summary.created++;
|
|
2821
2176
|
}
|
|
2822
2177
|
else {
|
|
2823
2178
|
summary.updated++;
|
|
2824
2179
|
}
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
2180
|
+
await importExportedState(client, appId, docDir, documentId);
|
|
2181
|
+
await uploadExportedBlobs(client, appId, documentId, docDir, exportedBlobsOf(docDir), summary);
|
|
2182
|
+
// Permissions are exported for reference but not restored during import.
|
|
2183
|
+
// The importing admin is the new owner; sharing is managed in the target app.
|
|
2184
|
+
await restoreExportedAliases(client, appId, documentId, metadata, ownerUserId, aliasMode, summary);
|
|
2185
|
+
info(`Imported document ${documentId} (${metadata.title || "untitled"})`);
|
|
2186
|
+
}
|
|
2187
|
+
/** Merge the exported Yjs state into the document, when the export has one. */
|
|
2188
|
+
async function importExportedState(client, appId, docDir, documentId) {
|
|
2189
|
+
const yjsPath = path.join(docDir, "document.yjs");
|
|
2190
|
+
if (!fs.existsSync(yjsPath))
|
|
2191
|
+
return;
|
|
2192
|
+
const stateBase64 = fs.readFileSync(yjsPath).toString("base64");
|
|
2193
|
+
await retryingImportStep(`Importing the state of ${documentId}`, () => client.importDocumentState(appId, documentId, stateBase64));
|
|
2194
|
+
}
|
|
2195
|
+
/** The blobs the export lists whose bytes are in it. */
|
|
2196
|
+
function exportedBlobsOf(docDir) {
|
|
2197
|
+
const blobIndexPath = path.join(docDir, "blobs", "index.json");
|
|
2198
|
+
if (!fs.existsSync(blobIndexPath))
|
|
2199
|
+
return [];
|
|
2200
|
+
const blobIndex = JSON.parse(fs.readFileSync(blobIndexPath, "utf-8"));
|
|
2201
|
+
return (Array.isArray(blobIndex) ? blobIndex : []).filter((blob) => fs.existsSync(path.join(docDir, "blobs", `${blob.blobId}.bin`)));
|
|
2202
|
+
}
|
|
2203
|
+
/** Upload exported blobs under their own ids. */
|
|
2204
|
+
async function uploadExportedBlobs(client, appId, documentId, docDir, blobs, summary) {
|
|
2205
|
+
for (const blob of blobs) {
|
|
2206
|
+
const data = fs.readFileSync(path.join(docDir, "blobs", `${blob.blobId}.bin`));
|
|
2207
|
+
await retryingImportStep(`Uploading blob ${blob.blobId} of ${documentId}`, () => client.uploadBlob(appId, documentId, blob.blobId, data, {
|
|
2208
|
+
filename: blob.filename,
|
|
2209
|
+
contentType: blob.contentType,
|
|
2210
|
+
sha256: blob.sha256,
|
|
2211
|
+
}));
|
|
2212
|
+
summary.blobsUploaded++;
|
|
2829
2213
|
}
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
|
|
2214
|
+
}
|
|
2215
|
+
/** Restore the export's user-scoped aliases, when an owner was named. */
|
|
2216
|
+
async function restoreExportedAliases(client, appId, documentId, metadata, ownerUserId, aliasMode, summary) {
|
|
2217
|
+
if (!Array.isArray(metadata.aliases))
|
|
2218
|
+
return;
|
|
2219
|
+
for (const alias of metadata.aliases) {
|
|
2220
|
+
if (alias.aliasScope !== "user")
|
|
2221
|
+
continue;
|
|
2222
|
+
if (!ownerUserId) {
|
|
2223
|
+
summary.aliasesSkipped++;
|
|
2224
|
+
continue;
|
|
2837
2225
|
}
|
|
2838
|
-
|
|
2839
|
-
|
|
2840
|
-
|
|
2841
|
-
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
});
|
|
2852
|
-
summary.blobsUploaded++;
|
|
2853
|
-
}
|
|
2226
|
+
try {
|
|
2227
|
+
await retryingImportStep(`Setting alias ${alias.aliasKey} of ${documentId}`, () => client.setDocumentAlias(appId, alias.aliasScope, alias.aliasKey, documentId, ownerUserId, aliasMode === "skip" // mustNotExist: true for skip mode, false for overwrite
|
|
2228
|
+
));
|
|
2229
|
+
summary.aliasesSet++;
|
|
2230
|
+
}
|
|
2231
|
+
catch (failure) {
|
|
2232
|
+
// An alias the user already holds is the answer `--aliases skip` asks
|
|
2233
|
+
// for, and it is left where it is. Any other failure leaves the
|
|
2234
|
+
// document without an alias its export holds: it is incomplete, and
|
|
2235
|
+
// reported as such (#4148).
|
|
2236
|
+
if (failure?.statusCode !== 409)
|
|
2237
|
+
throw failure;
|
|
2238
|
+
summary.aliasesSkipped++;
|
|
2854
2239
|
}
|
|
2855
2240
|
}
|
|
2856
|
-
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
2860
|
-
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
|
|
2864
|
-
summary.aliasesSkipped++;
|
|
2865
|
-
continue;
|
|
2866
|
-
}
|
|
2867
|
-
try {
|
|
2868
|
-
await client.setDocumentAlias(appId, alias.aliasScope, alias.aliasKey, documentId, ownerUserId, aliasMode === "skip" // mustNotExist: true for skip mode, false for overwrite
|
|
2869
|
-
);
|
|
2870
|
-
summary.aliasesSet++;
|
|
2871
|
-
}
|
|
2872
|
-
catch {
|
|
2873
|
-
summary.aliasesSkipped++;
|
|
2874
|
-
}
|
|
2875
|
-
}
|
|
2241
|
+
}
|
|
2242
|
+
/** "its state and 2 blobs", for the line that reports a completion. */
|
|
2243
|
+
function describeImportGaps(gaps) {
|
|
2244
|
+
const parts = [];
|
|
2245
|
+
if (gaps.state)
|
|
2246
|
+
parts.push("its state");
|
|
2247
|
+
if (gaps.blobs.length > 0) {
|
|
2248
|
+
parts.push(`${gaps.blobs.length} blob${gaps.blobs.length === 1 ? "" : "s"}`);
|
|
2876
2249
|
}
|
|
2877
|
-
|
|
2250
|
+
if (gaps.aliases.length > 0) {
|
|
2251
|
+
parts.push(`${gaps.aliases.length} alias${gaps.aliases.length === 1 ? "" : "es"}`);
|
|
2252
|
+
}
|
|
2253
|
+
return parts.join(" and ");
|
|
2878
2254
|
}
|
|
2879
|
-
/**
|
|
2880
|
-
|
|
2881
|
-
|
|
2882
|
-
|
|
2255
|
+
/** A Yjs update that holds nothing: no items and no deletions. */
|
|
2256
|
+
function isEmptyYjsUpdate(update) {
|
|
2257
|
+
return update.length === 0 || (update.length === 2 && update[0] === 0 && update[1] === 0);
|
|
2258
|
+
}
|
|
2259
|
+
/** The export's Yjs state, or null when it has none or an empty one. */
|
|
2260
|
+
function exportedStateOf(docDir) {
|
|
2261
|
+
const yjsPath = path.join(docDir, "document.yjs");
|
|
2262
|
+
if (!fs.existsSync(yjsPath))
|
|
2263
|
+
return null;
|
|
2264
|
+
const state = fs.readFileSync(yjsPath);
|
|
2265
|
+
return isEmptyYjsUpdate(state) ? null : state;
|
|
2266
|
+
}
|
|
2267
|
+
/**
|
|
2268
|
+
* Whether `held` already contains all of `exported`: for each client the
|
|
2269
|
+
* export wrote as, the document holds at least as far as the export does, and
|
|
2270
|
+
* it holds every deletion the export does. A deletion advances no clock, so
|
|
2271
|
+
* the clocks alone take a document read before a deletion for one after it.
|
|
2272
|
+
*/
|
|
2273
|
+
function holdsYjsUpdate(held, exported) {
|
|
2274
|
+
const heldClocks = Y.decodeStateVector(Y.encodeStateVectorFromUpdate(held));
|
|
2275
|
+
const exportedClocks = Y.decodeStateVector(Y.encodeStateVectorFromUpdate(exported));
|
|
2276
|
+
for (const [clientId, clock] of exportedClocks) {
|
|
2277
|
+
if ((heldClocks.get(clientId) ?? 0) < clock)
|
|
2278
|
+
return false;
|
|
2279
|
+
}
|
|
2280
|
+
// Every item the export deletes is held, so applying it changes the held
|
|
2281
|
+
// snapshot only if it deletes one the document has not.
|
|
2282
|
+
const doc = new Y.Doc();
|
|
2283
|
+
Y.applyUpdate(doc, held);
|
|
2284
|
+
const before = Y.snapshot(doc);
|
|
2285
|
+
Y.applyUpdate(doc, exported);
|
|
2286
|
+
return Y.equalSnapshots(before, Y.snapshot(doc));
|
|
2287
|
+
}
|
|
2288
|
+
/**
|
|
2289
|
+
* Read what an existing document lacks of its export (#4148): the gaps, or
|
|
2290
|
+
* null when there is nothing a re-run may write.
|
|
2291
|
+
*
|
|
2292
|
+
* A document is only completed when it is the one an earlier run started: it
|
|
2293
|
+
* holds no state at all — what `createDocument` leaves, and what a failed
|
|
2294
|
+
* `import/state` leaves, since the server stores the merge before it applies
|
|
2295
|
+
* it — or it already holds the exported state, and then only its blobs can be
|
|
2296
|
+
* missing. A document holding state of its own is not written to at all,
|
|
2297
|
+
* neither merged into nor given blobs or aliases: that is `--overwrite`'s
|
|
2298
|
+
* decision, not a re-run's. Blobs are compared by id against every page the
|
|
2299
|
+
* document lists. When `aliasOwner` is given, the export's user aliases are
|
|
2300
|
+
* read as well: one the owner holds for another document counts as missing
|
|
2301
|
+
* only when the alias mode would move it.
|
|
2302
|
+
*/
|
|
2303
|
+
async function findImportGaps(client, appId, docDir, documentId, aliasOwner) {
|
|
2304
|
+
let state = false;
|
|
2305
|
+
const exported = exportedBlobsOf(docDir);
|
|
2306
|
+
const exportedState = exportedStateOf(docDir);
|
|
2307
|
+
// An export with no state and no blobs can still carry a root's aliases.
|
|
2308
|
+
const exportsUserAliases = (Array.isArray(aliasOwner?.exported) ? aliasOwner.exported : [])
|
|
2309
|
+
.some((alias) => alias?.aliasScope === "user");
|
|
2310
|
+
if (!exportedState && exported.length === 0 && !exportsUserAliases)
|
|
2311
|
+
return null;
|
|
2312
|
+
const held = await retryingImportStep(`Reading the state of ${documentId}`, () => client.exportDocumentState(appId, documentId));
|
|
2313
|
+
const heldState = Buffer.from(String(held?.state ?? ""), "base64");
|
|
2314
|
+
if (isEmptyYjsUpdate(heldState)) {
|
|
2315
|
+
state = exportedState !== null;
|
|
2316
|
+
}
|
|
2317
|
+
else if (!exportedState || !holdsYjsUpdate(heldState, exportedState)) {
|
|
2318
|
+
return null;
|
|
2319
|
+
}
|
|
2320
|
+
let blobs = [];
|
|
2321
|
+
if (exported.length > 0) {
|
|
2322
|
+
const listed = await listedBlobIds(client, appId, documentId);
|
|
2323
|
+
blobs = exported.filter((blob) => !listed.has(blob.blobId));
|
|
2324
|
+
}
|
|
2325
|
+
const aliases = aliasOwner
|
|
2326
|
+
? await findMissingAliases(client, appId, documentId, aliasOwner)
|
|
2327
|
+
: [];
|
|
2328
|
+
return state || blobs.length > 0 || aliases.length > 0
|
|
2329
|
+
? { state, blobs, aliases }
|
|
2330
|
+
: null;
|
|
2331
|
+
}
|
|
2332
|
+
/** The form the server stores an alias key in. */
|
|
2333
|
+
function normalizedAliasKey(aliasKey) {
|
|
2334
|
+
return String(aliasKey ?? "").trim().toLowerCase();
|
|
2335
|
+
}
|
|
2336
|
+
/**
|
|
2337
|
+
* The exported user aliases that do not point at the document for their
|
|
2338
|
+
* owner. One the owner holds for another document is only missing under
|
|
2339
|
+
* `--aliases overwrite`: under `skip` the restore leaves it where it is, so a
|
|
2340
|
+
* re-run would have nothing to write for it.
|
|
2341
|
+
*/
|
|
2342
|
+
async function findMissingAliases(client, appId, documentId, owner) {
|
|
2343
|
+
const wanted = (Array.isArray(owner.exported) ? owner.exported : []).filter((alias) => alias?.aliasScope === "user");
|
|
2344
|
+
if (wanted.length === 0)
|
|
2345
|
+
return [];
|
|
2346
|
+
const listed = await retryingImportStep(`Listing the aliases of ${documentId}`, () => client.listDocumentAliases(appId, documentId));
|
|
2347
|
+
const held = new Set((Array.isArray(listed) ? listed : (listed?.aliases ?? []))
|
|
2348
|
+
.filter((alias) => (alias?.scope || alias?.aliasScope) === "user" && alias?.userId === owner.userId)
|
|
2349
|
+
.map((alias) => normalizedAliasKey(alias.aliasKey)));
|
|
2350
|
+
const missing = [];
|
|
2351
|
+
for (const alias of wanted) {
|
|
2352
|
+
if (held.has(normalizedAliasKey(alias.aliasKey)))
|
|
2353
|
+
continue;
|
|
2354
|
+
if (owner.aliasMode === "skip") {
|
|
2355
|
+
const current = await retryingImportStep(`Reading alias ${alias.aliasKey}`, () => client.getDocumentAlias(appId, "user", alias.aliasKey, owner.userId));
|
|
2356
|
+
if (current)
|
|
2357
|
+
continue;
|
|
2358
|
+
}
|
|
2359
|
+
missing.push(alias);
|
|
2360
|
+
}
|
|
2361
|
+
return missing;
|
|
2362
|
+
}
|
|
2363
|
+
/** Every blob id a document lists, read to the last page. */
|
|
2364
|
+
async function listedBlobIds(client, appId, documentId) {
|
|
2365
|
+
const ids = new Set();
|
|
2366
|
+
let cursor;
|
|
2367
|
+
for (;;) {
|
|
2368
|
+
const page = await retryingImportStep(`Listing the blobs of ${documentId}`, () => client.listDocumentBlobs(appId, documentId, cursor ? { cursor } : undefined));
|
|
2369
|
+
const items = Array.isArray(page) ? page : (page?.items ?? []);
|
|
2370
|
+
for (const item of items) {
|
|
2371
|
+
if (item?.blobId)
|
|
2372
|
+
ids.add(String(item.blobId));
|
|
2373
|
+
}
|
|
2374
|
+
if (Array.isArray(page) || !page?.nextCursor)
|
|
2375
|
+
return ids;
|
|
2376
|
+
cursor = String(page.nextCursor);
|
|
2377
|
+
}
|
|
2378
|
+
}
|
|
2379
|
+
/**
|
|
2380
|
+
* Restore a root-marked export into the target user's own root document
|
|
2381
|
+
* (#3135): its state is merged into the target user's root.
|
|
2382
|
+
*
|
|
2883
2383
|
* The exported id is never created and never looked up: it is the source
|
|
2884
2384
|
* user's root id, which means nothing in this app. Everything — the Yjs state,
|
|
2885
2385
|
* the blobs, the user-scoped aliases — is written against the target user's
|
|
@@ -2890,12 +2390,15 @@ async function importSingleDocument(client, appId, docDir, overwrite, aliasMode,
|
|
|
2890
2390
|
* holds survive, and a key both sides set resolves by Yjs's own conflict rule,
|
|
2891
2391
|
* which guarantees neither source-wins nor target-wins. That is why merging
|
|
2892
2392
|
* into an existing root needs `--overwrite` — the default must not quietly
|
|
2893
|
-
* rewrite a user's live preferences.
|
|
2393
|
+
* rewrite a user's live preferences. Without it, an existing legacy root is
|
|
2394
|
+
* only completed (#4148): one an earlier run of this import started and did
|
|
2395
|
+
* not finish gets what it lacks, exactly as a legacy document does. An
|
|
2396
|
+
* existing large root is skipped.
|
|
2894
2397
|
*/
|
|
2895
|
-
async function importRootDocument(client, appId, docDir, metadata, plan, overwrite, aliasMode, dryRun, summary) {
|
|
2398
|
+
async function importRootDocument(client, appId, docDir, metadata, plan, overwrite, aliasMode, dryRun, summary, progress) {
|
|
2896
2399
|
const exportedDocumentId = metadata.documentId || path.basename(docDir);
|
|
2897
2400
|
const wouldSkip = dryRun ? "[dry-run] Would skip" : "Skipping";
|
|
2898
|
-
const report = (action, targetRootDocId, reason) => {
|
|
2401
|
+
const report = (action, targetRootDocId, reason, targetFormat) => {
|
|
2899
2402
|
summary.rootDocuments.push({
|
|
2900
2403
|
exportedDocumentId,
|
|
2901
2404
|
owner: plan?.owner ?? "",
|
|
@@ -2913,31 +2416,54 @@ async function importRootDocument(client, appId, docDir, metadata, plan, overwri
|
|
|
2913
2416
|
report("refused", undefined, reason);
|
|
2914
2417
|
return;
|
|
2915
2418
|
}
|
|
2419
|
+
let targetRootDocId = plan.targetRootDocId;
|
|
2420
|
+
// What a root an earlier run started still lacks (#4148), when completing it.
|
|
2421
|
+
let gaps = null;
|
|
2916
2422
|
if (plan.action === "skip") {
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
|
|
2923
|
-
|
|
2423
|
+
// The root exists and `--overwrite` was not given. A root an earlier run
|
|
2424
|
+
// started and did not finish is completed, by the same rule as any
|
|
2425
|
+
// document: it holds no state, or already holds the exported state. A
|
|
2426
|
+
// root holding state of its own — its user's — is never merged into.
|
|
2427
|
+
progress.exists = true;
|
|
2428
|
+
const found = await findImportGaps(client, appId, docDir, targetRootDocId, {
|
|
2429
|
+
userId: plan.targetUserId,
|
|
2430
|
+
aliasMode,
|
|
2431
|
+
exported: metadata.aliases,
|
|
2432
|
+
});
|
|
2433
|
+
if (!found) {
|
|
2434
|
+
const reason = plan.reason ||
|
|
2435
|
+
`the root document of ${plan.owner} already exists ` +
|
|
2436
|
+
`(${targetRootDocId}); use --overwrite to merge the exported ` +
|
|
2437
|
+
`content into it`;
|
|
2438
|
+
warn(`${wouldSkip} ${exportedDocumentId}: ${reason}`);
|
|
2439
|
+
summary.skipped++;
|
|
2440
|
+
report("skipped", targetRootDocId, reason, plan.targetFormat);
|
|
2441
|
+
return;
|
|
2442
|
+
}
|
|
2443
|
+
gaps = found;
|
|
2924
2444
|
}
|
|
2925
2445
|
if (dryRun) {
|
|
2926
|
-
if (
|
|
2446
|
+
if (gaps) {
|
|
2447
|
+
info(`[dry-run] Would complete root document export ${exportedDocumentId} ` +
|
|
2448
|
+
`in ${plan.owner}'s root document ${targetRootDocId}: it is missing ` +
|
|
2449
|
+
describeImportGaps(gaps));
|
|
2450
|
+
summary.completed++;
|
|
2451
|
+
report("completed", targetRootDocId);
|
|
2452
|
+
}
|
|
2453
|
+
else if (plan.action === "create") {
|
|
2927
2454
|
info(`[dry-run] Would create the root document for ${plan.owner} and ` +
|
|
2928
2455
|
`apply root document export ${exportedDocumentId} to it`);
|
|
2929
2456
|
summary.created++;
|
|
2930
|
-
report("created");
|
|
2457
|
+
report("created", undefined, undefined, plan.targetFormat);
|
|
2931
2458
|
}
|
|
2932
2459
|
else {
|
|
2933
|
-
info(`[dry-run] Would apply root document export ${exportedDocumentId}
|
|
2934
|
-
|
|
2460
|
+
info(`[dry-run] Would apply root document export ${exportedDocumentId} ` +
|
|
2461
|
+
`to ${plan.owner}'s root document ${plan.targetRootDocId}`);
|
|
2935
2462
|
summary.updated++;
|
|
2936
|
-
report("applied", plan.targetRootDocId);
|
|
2463
|
+
report("applied", plan.targetRootDocId, undefined, plan.targetFormat);
|
|
2937
2464
|
}
|
|
2938
2465
|
return;
|
|
2939
2466
|
}
|
|
2940
|
-
let targetRootDocId = plan.targetRootDocId;
|
|
2941
2467
|
let created = false;
|
|
2942
2468
|
if (plan.action === "create") {
|
|
2943
2469
|
// The server's own get-or-create, and the only place a root document is
|
|
@@ -2951,94 +2477,918 @@ async function importRootDocument(client, appId, docDir, metadata, plan, overwri
|
|
|
2951
2477
|
`for ${plan.owner}, so nothing was imported for it.`);
|
|
2952
2478
|
}
|
|
2953
2479
|
created = ensured.created === true;
|
|
2480
|
+
// The root the server returned is the truth, not the plan's prediction
|
|
2481
|
+
// (#4195): the app's setting can change, or a sign-in can mint the root,
|
|
2482
|
+
// between the plan and this call. Decided before the concurrent-sign-in
|
|
2483
|
+
// skip, so a root of the other format is refused, never offered to
|
|
2484
|
+
// --overwrite.
|
|
2485
|
+
const minted = decideRootImport({
|
|
2486
|
+
exportFormat: plan.exportFormat,
|
|
2487
|
+
existingRootFormat: rootDocumentFormatOf(ensured.documentFormat),
|
|
2488
|
+
predictedFormat: plan.targetFormat,
|
|
2489
|
+
appSetting: null,
|
|
2490
|
+
overwrite: true,
|
|
2491
|
+
owner: plan.owner,
|
|
2492
|
+
targetRootDocId,
|
|
2493
|
+
});
|
|
2494
|
+
if (minted.outcome === "refuse") {
|
|
2495
|
+
warn(`Skipping ${exportedDocumentId}: ${minted.reason}`);
|
|
2496
|
+
summary.skipped++;
|
|
2497
|
+
report("refused", targetRootDocId, minted.reason, minted.targetFormat);
|
|
2498
|
+
return;
|
|
2499
|
+
}
|
|
2954
2500
|
if (!created && !overwrite) {
|
|
2955
2501
|
const reason = `the root document of ${plan.owner} (${targetRootDocId}) was created ` +
|
|
2956
2502
|
`by a concurrent sign-in while this import was running; use ` +
|
|
2957
2503
|
`--overwrite to merge the exported content into it`;
|
|
2958
2504
|
warn(`Skipping ${exportedDocumentId}: ${reason}`);
|
|
2959
2505
|
summary.skipped++;
|
|
2960
|
-
report("skipped", targetRootDocId, reason);
|
|
2506
|
+
report("skipped", targetRootDocId, reason, minted.targetFormat);
|
|
2961
2507
|
return;
|
|
2962
2508
|
}
|
|
2963
2509
|
}
|
|
2510
|
+
progress.exists = true;
|
|
2964
2511
|
// Only content moves. The target root keeps its own id, title, tags and
|
|
2965
2512
|
// permissions — the ensure path already granted the owner read-write.
|
|
2966
|
-
|
|
2967
|
-
|
|
2968
|
-
await client.importDocumentState(appId, targetRootDocId, fs.readFileSync(yjsPath).toString("base64"));
|
|
2969
|
-
}
|
|
2970
|
-
const blobIndexPath = path.join(docDir, "blobs", "index.json");
|
|
2971
|
-
if (fs.existsSync(blobIndexPath)) {
|
|
2972
|
-
const blobIndex = JSON.parse(fs.readFileSync(blobIndexPath, "utf-8"));
|
|
2973
|
-
for (const blob of blobIndex) {
|
|
2974
|
-
const blobPath = path.join(docDir, "blobs", `${blob.blobId}.bin`);
|
|
2975
|
-
if (!fs.existsSync(blobPath))
|
|
2976
|
-
continue;
|
|
2977
|
-
// The blobId is preserved: a document's own references travel by blobId,
|
|
2978
|
-
// so they resolve inside the target root as they did in the source.
|
|
2979
|
-
await client.uploadBlob(appId, targetRootDocId, blob.blobId, fs.readFileSync(blobPath), {
|
|
2980
|
-
filename: blob.filename,
|
|
2981
|
-
contentType: blob.contentType,
|
|
2982
|
-
sha256: blob.sha256,
|
|
2983
|
-
});
|
|
2984
|
-
summary.blobsUploaded++;
|
|
2985
|
-
}
|
|
2513
|
+
if (!gaps || gaps.state) {
|
|
2514
|
+
await importExportedState(client, appId, docDir, targetRootDocId);
|
|
2986
2515
|
}
|
|
2516
|
+
// The blobId is preserved: a document's own references travel by blobId,
|
|
2517
|
+
// so they resolve inside the target root as they did in the source.
|
|
2518
|
+
await uploadExportedBlobs(client, appId, targetRootDocId, docDir, gaps ? gaps.blobs : exportedBlobsOf(docDir), summary);
|
|
2987
2519
|
// A root document's user-scoped aliases belong to its owner by definition,
|
|
2988
2520
|
// so they are restored without `--owner` — the target user is already known.
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
catch {
|
|
2998
|
-
summary.aliasesSkipped++;
|
|
2999
|
-
}
|
|
3000
|
-
}
|
|
2521
|
+
// A completion restores only the ones the root is missing.
|
|
2522
|
+
await restoreExportedAliases(client, appId, targetRootDocId, gaps ? { ...metadata, aliases: gaps.aliases } : metadata, plan.targetUserId, aliasMode, summary);
|
|
2523
|
+
if (gaps) {
|
|
2524
|
+
summary.completed++;
|
|
2525
|
+
report("completed", targetRootDocId);
|
|
2526
|
+
info(`Completed root document export ${exportedDocumentId} in ` +
|
|
2527
|
+
`${plan.owner}'s root document ${targetRootDocId}: imported ` +
|
|
2528
|
+
describeImportGaps(gaps));
|
|
3001
2529
|
}
|
|
3002
|
-
if (created) {
|
|
2530
|
+
else if (created) {
|
|
3003
2531
|
summary.created++;
|
|
3004
|
-
report("created", targetRootDocId);
|
|
2532
|
+
report("created", targetRootDocId, undefined, plan.targetFormat);
|
|
3005
2533
|
info(`Created root document ${targetRootDocId} for ${plan.owner} and applied ` +
|
|
3006
2534
|
`root document export ${exportedDocumentId} to it`);
|
|
3007
2535
|
}
|
|
3008
2536
|
else {
|
|
3009
2537
|
summary.updated++;
|
|
3010
|
-
report("applied", targetRootDocId);
|
|
3011
|
-
info(`Applied root document export ${exportedDocumentId} to
|
|
3012
|
-
|
|
2538
|
+
report("applied", targetRootDocId, undefined, plan.targetFormat);
|
|
2539
|
+
info(`Applied root document export ${exportedDocumentId} to ` +
|
|
2540
|
+
`${plan.owner}'s root document ${targetRootDocId}`);
|
|
2541
|
+
}
|
|
2542
|
+
}
|
|
2543
|
+
/** The aggregate operations `records aggregate` takes. */
|
|
2544
|
+
const AGGREGATE_OPS = ["count", "sum", "avg", "min", "max"];
|
|
2545
|
+
// ---- The records, schema and inspect verbs' bodies (#4696) ----
|
|
2546
|
+
// Shared by the `documents` verbs and their `db3` twins (`./db3.ts`): each
|
|
2547
|
+
// sends its request through `routes` and prints the answer, so both nouns
|
|
2548
|
+
// print the same thing for the same answer. The caller has resolved the app
|
|
2549
|
+
// (where the body does not) and read its own flags, `--document-format`
|
|
2550
|
+
// included.
|
|
2551
|
+
/** `records query`: one page of a model's records, as a table or `--json`. */
|
|
2552
|
+
export async function queryRecordsAction(routes, resolvedAppId, documentId, modelName, options, documentFormat) {
|
|
2553
|
+
const filter = parseFilterOptions(options);
|
|
2554
|
+
// The JSON is checked here; its shape (`{ field: 1 | -1 }`) is the
|
|
2555
|
+
// route's to refuse, with its own `Invalid sort` message.
|
|
2556
|
+
let sort;
|
|
2557
|
+
if (options.sort !== undefined) {
|
|
2558
|
+
try {
|
|
2559
|
+
sort = JSON.parse(options.sort);
|
|
2560
|
+
}
|
|
2561
|
+
catch {
|
|
2562
|
+
error("Invalid --sort JSON.");
|
|
2563
|
+
process.exit(1);
|
|
2564
|
+
}
|
|
2565
|
+
}
|
|
2566
|
+
// The same for `--include`: its specs are the route's to refuse.
|
|
2567
|
+
let include;
|
|
2568
|
+
if (options.include !== undefined) {
|
|
2569
|
+
try {
|
|
2570
|
+
include = JSON.parse(options.include);
|
|
2571
|
+
}
|
|
2572
|
+
catch {
|
|
2573
|
+
error("Invalid --include JSON.");
|
|
2574
|
+
process.exit(1);
|
|
2575
|
+
}
|
|
2576
|
+
}
|
|
2577
|
+
const client = new ApiClient();
|
|
2578
|
+
try {
|
|
2579
|
+
const result = await routes.query(client, resolvedAppId, documentId, modelName, { filter, sort, limit: options.limit, cursor: options.cursor, documentFormat, include });
|
|
2580
|
+
if (options.json) {
|
|
2581
|
+
json(result);
|
|
2582
|
+
return;
|
|
2583
|
+
}
|
|
2584
|
+
const items = result.items;
|
|
2585
|
+
if (items.length === 0) {
|
|
2586
|
+
info("No records found.");
|
|
2587
|
+
return;
|
|
2588
|
+
}
|
|
2589
|
+
const allKeys = new Set();
|
|
2590
|
+
for (const rec of items) {
|
|
2591
|
+
for (const key of Object.keys(rec))
|
|
2592
|
+
allKeys.add(key);
|
|
2593
|
+
}
|
|
2594
|
+
const orderedKeys = [];
|
|
2595
|
+
if (allKeys.has("id")) {
|
|
2596
|
+
orderedKeys.push("id");
|
|
2597
|
+
allKeys.delete("id");
|
|
2598
|
+
}
|
|
2599
|
+
orderedKeys.push(...[...allKeys].sort());
|
|
2600
|
+
const columns = orderedKeys.map((key) => ({
|
|
2601
|
+
header: key.toUpperCase(),
|
|
2602
|
+
key,
|
|
2603
|
+
format: (v) => {
|
|
2604
|
+
if (v === null || v === undefined)
|
|
2605
|
+
return "—";
|
|
2606
|
+
if (typeof v === "object")
|
|
2607
|
+
return JSON.stringify(v);
|
|
2608
|
+
return String(v);
|
|
2609
|
+
},
|
|
2610
|
+
}));
|
|
2611
|
+
console.log(formatTable(items, columns));
|
|
2612
|
+
if (result.nextCursor) {
|
|
2613
|
+
// A cursor continues only the sort that minted it.
|
|
2614
|
+
const sortArg = sort !== undefined ? ` --sort ${quoteCursorArg(JSON.stringify(sort))}` : "";
|
|
2615
|
+
info(`More results available. Use --cursor ${result.nextCursor}${sortArg}`);
|
|
2616
|
+
}
|
|
2617
|
+
}
|
|
2618
|
+
catch (err) {
|
|
2619
|
+
// #3764 — the server's sentence, the stable code and the status, so a
|
|
2620
|
+
// refused records call is scriptable rather than only readable.
|
|
2621
|
+
error(describeRecordsFailure(err));
|
|
2622
|
+
process.exit(1);
|
|
2623
|
+
}
|
|
2624
|
+
}
|
|
2625
|
+
/** `records count`: how many of a model's records match the filter. */
|
|
2626
|
+
export async function countRecordsAction(routes, resolvedAppId, documentId, modelName, options, documentFormat) {
|
|
2627
|
+
const filter = parseFilterOptions(options);
|
|
2628
|
+
const client = new ApiClient();
|
|
2629
|
+
try {
|
|
2630
|
+
const result = await routes.count(client, resolvedAppId, documentId, modelName, { filter, documentFormat });
|
|
2631
|
+
if (options.json) {
|
|
2632
|
+
json(result);
|
|
2633
|
+
return;
|
|
2634
|
+
}
|
|
2635
|
+
console.log(` ${result.count}`);
|
|
2636
|
+
}
|
|
2637
|
+
catch (err) {
|
|
2638
|
+
// #3764 — the server's sentence, the stable code and the status, so a
|
|
2639
|
+
// refused records call is scriptable rather than only readable.
|
|
2640
|
+
error(describeRecordsFailure(err));
|
|
2641
|
+
process.exit(1);
|
|
2642
|
+
}
|
|
2643
|
+
}
|
|
2644
|
+
/** `records aggregate`: one operation over a model's records, optionally grouped. */
|
|
2645
|
+
export async function aggregateRecordsAction(routes, resolvedAppId, documentId, modelName, options, documentFormat) {
|
|
2646
|
+
if (!AGGREGATE_OPS.includes(options.op)) {
|
|
2647
|
+
error(`Invalid --op "${options.op}". Expected one of: ${AGGREGATE_OPS.join(", ")}.`);
|
|
2648
|
+
process.exit(1);
|
|
2649
|
+
}
|
|
2650
|
+
const operation = { type: options.op };
|
|
2651
|
+
if (options.op === "count") {
|
|
2652
|
+
if (options.field) {
|
|
2653
|
+
error("--field is not valid for --op count.");
|
|
2654
|
+
process.exit(1);
|
|
2655
|
+
}
|
|
2656
|
+
}
|
|
2657
|
+
else {
|
|
2658
|
+
if (!options.field) {
|
|
2659
|
+
error(`--op ${options.op} requires --field.`);
|
|
2660
|
+
process.exit(1);
|
|
2661
|
+
}
|
|
2662
|
+
operation.field = options.field;
|
|
2663
|
+
}
|
|
2664
|
+
const filter = parseFilterOptions(options);
|
|
2665
|
+
const groupBy = options.groupBy || [];
|
|
2666
|
+
const client = new ApiClient();
|
|
2667
|
+
try {
|
|
2668
|
+
const response = await routes.aggregate(client, resolvedAppId, documentId, modelName, { operations: [operation], groupBy, filter }, { documentFormat });
|
|
2669
|
+
if (options.json) {
|
|
2670
|
+
json(response);
|
|
2671
|
+
return;
|
|
2672
|
+
}
|
|
2673
|
+
// The route answers rows (#4265): one key/value line when
|
|
2674
|
+
// ungrouped, a table with one column per group key when grouped.
|
|
2675
|
+
const rendered = renderAggregateResult(response?.result, {
|
|
2676
|
+
groupBy,
|
|
2677
|
+
op: options.op,
|
|
2678
|
+
field: options.field,
|
|
2679
|
+
});
|
|
2680
|
+
if (rendered.kind === "scalar") {
|
|
2681
|
+
keyValue(rendered.label, rendered.value);
|
|
2682
|
+
}
|
|
2683
|
+
else {
|
|
2684
|
+
console.log(rendered.text);
|
|
2685
|
+
}
|
|
2686
|
+
}
|
|
2687
|
+
catch (err) {
|
|
2688
|
+
// #3764 — the server's sentence, the stable code and the status, so a
|
|
2689
|
+
// refused records call is scriptable rather than only readable.
|
|
2690
|
+
error(describeRecordsFailure(err));
|
|
2691
|
+
process.exit(1);
|
|
2692
|
+
}
|
|
2693
|
+
}
|
|
2694
|
+
/** `records save`: create or replace a record with the parsed `data`. */
|
|
2695
|
+
export async function saveRecordAction(routes, resolvedAppId, documentId, modelName, data, options, documentFormat) {
|
|
2696
|
+
const recordId = options.id || ulid();
|
|
2697
|
+
const writeOptions = options.upsertOn ? { upsertOn: options.upsertOn } : undefined;
|
|
2698
|
+
const client = new ApiClient();
|
|
2699
|
+
try {
|
|
2700
|
+
const result = await routes.save(client, resolvedAppId, documentId, modelName, { id: recordId, data, ...(writeOptions ? { options: writeOptions } : {}) }, { documentFormat });
|
|
2701
|
+
if (options.json) {
|
|
2702
|
+
json(result);
|
|
2703
|
+
return;
|
|
2704
|
+
}
|
|
2705
|
+
success(`Record saved: ${result.record?.id ?? recordId}`);
|
|
2706
|
+
}
|
|
2707
|
+
catch (err) {
|
|
2708
|
+
// #3764 — the server's sentence, the stable code and the status, so a
|
|
2709
|
+
// refused records call is scriptable rather than only readable.
|
|
2710
|
+
error(describeRecordsFailure(err));
|
|
2711
|
+
process.exit(1);
|
|
2712
|
+
}
|
|
2713
|
+
}
|
|
2714
|
+
/** `records patch`: merge the parsed `data` into a record. */
|
|
2715
|
+
export async function patchRecordAction(routes, resolvedAppId, documentId, modelName, recordId, data, options, documentFormat) {
|
|
2716
|
+
const client = new ApiClient();
|
|
2717
|
+
try {
|
|
2718
|
+
const result = await routes.patch(client, resolvedAppId, documentId, modelName, recordId, { data }, { documentFormat });
|
|
2719
|
+
if (options.json) {
|
|
2720
|
+
json(result);
|
|
2721
|
+
return;
|
|
2722
|
+
}
|
|
2723
|
+
success(`Record patched: ${recordId}`);
|
|
2724
|
+
}
|
|
2725
|
+
catch (err) {
|
|
2726
|
+
// #3764 — the server's sentence, the stable code and the status, so a
|
|
2727
|
+
// refused records call is scriptable rather than only readable.
|
|
2728
|
+
error(describeRecordsFailure(err));
|
|
2729
|
+
process.exit(1);
|
|
2730
|
+
}
|
|
2731
|
+
}
|
|
2732
|
+
/** `records delete`: delete a record, confirming first unless `--yes`. */
|
|
2733
|
+
export async function deleteRecordAction(routes, resolvedAppId, documentId, modelName, recordId, options, documentFormat) {
|
|
2734
|
+
if (!options.yes) {
|
|
2735
|
+
const confirmed = await confirmPrompt(`Delete record ${recordId} from ${modelName} in ${routes.noun} ${documentId}?`);
|
|
2736
|
+
if (!confirmed) {
|
|
2737
|
+
info("Cancelled.");
|
|
2738
|
+
return;
|
|
2739
|
+
}
|
|
2740
|
+
}
|
|
2741
|
+
const client = new ApiClient();
|
|
2742
|
+
try {
|
|
2743
|
+
const result = await routes.delete(client, resolvedAppId, documentId, modelName, recordId, { documentFormat });
|
|
2744
|
+
if (options.json) {
|
|
2745
|
+
json(result);
|
|
2746
|
+
return;
|
|
2747
|
+
}
|
|
2748
|
+
// A delete of a missing record is a silent no-op server-side, so
|
|
2749
|
+
// this reports the request outcome, not record existence.
|
|
2750
|
+
success(`Record deleted: ${recordId}`);
|
|
2751
|
+
}
|
|
2752
|
+
catch (err) {
|
|
2753
|
+
// #3764 — the server's sentence, the stable code and the status, so a
|
|
2754
|
+
// refused records call is scriptable rather than only readable.
|
|
2755
|
+
error(describeRecordsFailure(err));
|
|
2756
|
+
process.exit(1);
|
|
2757
|
+
}
|
|
2758
|
+
}
|
|
2759
|
+
/** `records bulk`: apply an operations blob atomically, confirming first unless `--yes`. */
|
|
2760
|
+
export async function bulkRecordsAction(routes, documentId, options, documentFormat) {
|
|
2761
|
+
const parsed = parseDataOption(options, "operations blob", {
|
|
2762
|
+
allowArray: true,
|
|
2763
|
+
});
|
|
2764
|
+
const operations = Array.isArray(parsed) ? parsed : parsed?.operations;
|
|
2765
|
+
if (!Array.isArray(operations) || operations.length === 0) {
|
|
2766
|
+
error("--data-file must contain { operations: [...] } or a non-empty operations array.");
|
|
2767
|
+
process.exit(1);
|
|
2768
|
+
}
|
|
2769
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
2770
|
+
if (!options.yes) {
|
|
2771
|
+
const confirmed = await confirmPrompt(`Apply ${operations.length} operation(s) atomically to ${routes.noun} ${documentId}?`);
|
|
2772
|
+
if (!confirmed) {
|
|
2773
|
+
info("Cancelled.");
|
|
2774
|
+
return;
|
|
2775
|
+
}
|
|
2776
|
+
}
|
|
2777
|
+
const client = new ApiClient();
|
|
2778
|
+
try {
|
|
2779
|
+
const result = await routes.bulk(client, resolvedAppId, documentId, operations, { documentFormat });
|
|
2780
|
+
if (options.json) {
|
|
2781
|
+
json(result);
|
|
2782
|
+
return;
|
|
2783
|
+
}
|
|
2784
|
+
success(describeBulkResult(result));
|
|
2785
|
+
}
|
|
2786
|
+
catch (err) {
|
|
2787
|
+
const report = describeBulkFailure(err);
|
|
2788
|
+
if (options.json)
|
|
2789
|
+
json(report.envelope);
|
|
2790
|
+
else
|
|
2791
|
+
error(report.line);
|
|
2792
|
+
process.exit(1);
|
|
2793
|
+
}
|
|
2794
|
+
}
|
|
2795
|
+
/** `schema`: the models and fields as declared and observed, a table, the JSON or a schema.toml. */
|
|
2796
|
+
/** `records models` (shared with `db3 records models`, #4804): the model names the schema route answers. */
|
|
2797
|
+
export async function modelsAction(routes, documentId, options) {
|
|
2798
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
2799
|
+
const client = new ApiClient();
|
|
2800
|
+
try {
|
|
2801
|
+
const result = await routes.schema(client, resolvedAppId, documentId);
|
|
2802
|
+
const models = result?.modelNames ?? Object.keys(result?.schema?.models ?? {});
|
|
2803
|
+
if (options.json) {
|
|
2804
|
+
json(models);
|
|
2805
|
+
return;
|
|
2806
|
+
}
|
|
2807
|
+
if (models.length === 0) {
|
|
2808
|
+
info("No models found.");
|
|
2809
|
+
return;
|
|
2810
|
+
}
|
|
2811
|
+
for (const m of models) {
|
|
2812
|
+
console.log(` ${m}`);
|
|
2813
|
+
}
|
|
2814
|
+
}
|
|
2815
|
+
catch (err) {
|
|
2816
|
+
// #3764 — the server's sentence, the stable code and the status, so a
|
|
2817
|
+
// refused records call is scriptable rather than only readable.
|
|
2818
|
+
error(describeRecordsFailure(err));
|
|
2819
|
+
process.exit(1);
|
|
2820
|
+
}
|
|
2821
|
+
}
|
|
2822
|
+
/** `records describe` (shared with `db3 records describe`, #4804): one model's fields and indexes. */
|
|
2823
|
+
export async function describeAction(routes, documentId, modelName, options) {
|
|
2824
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
2825
|
+
const client = new ApiClient();
|
|
2826
|
+
try {
|
|
2827
|
+
const result = await routes.schema(client, resolvedAppId, documentId);
|
|
2828
|
+
const model = result?.schema?.models?.[modelName];
|
|
2829
|
+
if (!model) {
|
|
2830
|
+
if (options.json) {
|
|
2831
|
+
json(null);
|
|
2832
|
+
return;
|
|
2833
|
+
}
|
|
2834
|
+
info(`Model '${modelName}' not found in this ${routes.noun}.`);
|
|
2835
|
+
return;
|
|
2836
|
+
}
|
|
2837
|
+
const fieldsObj = model.fields ?? {};
|
|
2838
|
+
const rows = Object.entries(fieldsObj).map(([name, meta]) => ({
|
|
2839
|
+
field: name,
|
|
2840
|
+
type: meta?.type ?? "unknown",
|
|
2841
|
+
indexed: meta?.indexed ? "yes" : "",
|
|
2842
|
+
unique: meta?.unique ? "yes" : "",
|
|
2843
|
+
required: meta?.required ? "yes" : "",
|
|
2844
|
+
}));
|
|
2845
|
+
if (options.json) {
|
|
2846
|
+
json({ fields: fieldsObj, constraints: model.constraints ?? {} });
|
|
2847
|
+
return;
|
|
2848
|
+
}
|
|
2849
|
+
if (rows.length === 0) {
|
|
2850
|
+
info("No fields detected for this model.");
|
|
2851
|
+
return;
|
|
2852
|
+
}
|
|
2853
|
+
console.log(formatTable(rows, [
|
|
2854
|
+
{ header: "FIELD", key: "field", flex: true },
|
|
2855
|
+
{ header: "TYPE", key: "type" },
|
|
2856
|
+
{ header: "INDEXED", key: "indexed" },
|
|
2857
|
+
{ header: "UNIQUE", key: "unique" },
|
|
2858
|
+
{ header: "REQUIRED", key: "required" },
|
|
2859
|
+
]));
|
|
2860
|
+
}
|
|
2861
|
+
catch (err) {
|
|
2862
|
+
// #3764 — the server's sentence, the stable code and the status, so a
|
|
2863
|
+
// refused records call is scriptable rather than only readable.
|
|
2864
|
+
error(describeRecordsFailure(err));
|
|
2865
|
+
process.exit(1);
|
|
2866
|
+
}
|
|
2867
|
+
}
|
|
2868
|
+
/** `stats` (shared with `db3 stats`, #4804): record, model and blob counts and the size. */
|
|
2869
|
+
export async function statsAction(routes, documentId, options) {
|
|
2870
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
2871
|
+
const client = new ApiClient();
|
|
2872
|
+
try {
|
|
2873
|
+
const result = await routes.stats(client, resolvedAppId, documentId);
|
|
2874
|
+
if (options.json) {
|
|
2875
|
+
json(result);
|
|
2876
|
+
return;
|
|
2877
|
+
}
|
|
2878
|
+
keyValue(routes.noun === "db3" ? "db3 ID" : "Document ID", result.documentId);
|
|
2879
|
+
keyValue("Records", result.recordCount);
|
|
2880
|
+
keyValue("Models", result.modelCount);
|
|
2881
|
+
keyValue("Blobs", result.blobCount);
|
|
2882
|
+
// A large document's size is its records' stored JSON, exact; an
|
|
2883
|
+
// ordinary document's is an estimate (#4193).
|
|
2884
|
+
keyValue(result.sizeBasis === "records"
|
|
2885
|
+
? "Size (bytes, records)"
|
|
2886
|
+
: "Size (bytes, approx)", result.sizeBytes);
|
|
2887
|
+
keyValue("Last modified", result.lastModifiedAt ? formatDate(result.lastModifiedAt) : "—");
|
|
2888
|
+
}
|
|
2889
|
+
catch (err) {
|
|
2890
|
+
error(describeDocumentFailure(err));
|
|
2891
|
+
process.exit(1);
|
|
2892
|
+
}
|
|
2893
|
+
}
|
|
2894
|
+
export async function schemaAction(routes, documentId, options) {
|
|
2895
|
+
if (options.output && !options.toml) {
|
|
2896
|
+
error("--output writes the TOML: pass --toml with it.");
|
|
2897
|
+
process.exit(1);
|
|
2898
|
+
}
|
|
2899
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
2900
|
+
const client = new ApiClient();
|
|
2901
|
+
try {
|
|
2902
|
+
const answer = await routes.schema(client, resolvedAppId, documentId);
|
|
2903
|
+
if (options.json) {
|
|
2904
|
+
json(answer);
|
|
2905
|
+
return;
|
|
2906
|
+
}
|
|
2907
|
+
if (options.toml) {
|
|
2908
|
+
// Each field, constraint or model the export leaves out is one warn line.
|
|
2909
|
+
const toml = recordLogSchemaToToml(answer, { warn: (line) => warn(line) });
|
|
2910
|
+
if (options.output) {
|
|
2911
|
+
fs.writeFileSync(options.output, toml);
|
|
2912
|
+
success(`Wrote ${options.output}.`);
|
|
2913
|
+
return;
|
|
2914
|
+
}
|
|
2915
|
+
process.stdout.write(toml);
|
|
2916
|
+
return;
|
|
2917
|
+
}
|
|
2918
|
+
const rows = documentSchemaRows(answer);
|
|
2919
|
+
if (rows.length === 0) {
|
|
2920
|
+
info("No models found.");
|
|
2921
|
+
return;
|
|
2922
|
+
}
|
|
2923
|
+
console.log(formatTable(rows, [
|
|
2924
|
+
{ key: "model", header: "Model" },
|
|
2925
|
+
{ key: "field", header: "Field" },
|
|
2926
|
+
{ key: "type", header: "Type" },
|
|
2927
|
+
{ key: "kinds", header: "Kinds" },
|
|
2928
|
+
{ key: "counts", header: "Counts" },
|
|
2929
|
+
{ key: "hint", header: "Hint" },
|
|
2930
|
+
]));
|
|
2931
|
+
}
|
|
2932
|
+
catch (err) {
|
|
2933
|
+
error(describeRecordsFailure(err));
|
|
2934
|
+
process.exit(1);
|
|
2935
|
+
}
|
|
2936
|
+
}
|
|
2937
|
+
/** `inspect`: the room's state, or every row a bulk load refused. */
|
|
2938
|
+
export async function inspectAction(routes, documentId, options) {
|
|
2939
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
2940
|
+
const client = new ApiClient();
|
|
2941
|
+
try {
|
|
2942
|
+
if (options.refused !== undefined) {
|
|
2943
|
+
// Every page, in order: the room keeps each refused row with its code.
|
|
2944
|
+
const items = [];
|
|
2945
|
+
let cursor;
|
|
2946
|
+
do {
|
|
2947
|
+
const page = await routes.room(client, resolvedAppId, documentId, {
|
|
2948
|
+
refused: options.refused,
|
|
2949
|
+
limit: Number(options.limit),
|
|
2950
|
+
cursor,
|
|
2951
|
+
});
|
|
2952
|
+
items.push(...page.refused.items);
|
|
2953
|
+
cursor = page.refused.hasMore ? page.refused.nextCursor : undefined;
|
|
2954
|
+
} while (cursor);
|
|
2955
|
+
if (options.json) {
|
|
2956
|
+
json({ documentId, sessionId: options.refused, items });
|
|
2957
|
+
return;
|
|
2958
|
+
}
|
|
2959
|
+
keyValue("Document ID", documentId);
|
|
2960
|
+
keyValue("Session", options.refused);
|
|
2961
|
+
keyValue("Refused rows", items.length);
|
|
2962
|
+
for (const row of items)
|
|
2963
|
+
keyValue(...refusedRowLine(row));
|
|
2964
|
+
return;
|
|
2965
|
+
}
|
|
2966
|
+
const result = await routes.room(client, resolvedAppId, documentId);
|
|
2967
|
+
if (options.json) {
|
|
2968
|
+
json(result);
|
|
2969
|
+
return;
|
|
2970
|
+
}
|
|
2971
|
+
keyValue("Document ID", result.documentId);
|
|
2972
|
+
printDocumentFormat(result);
|
|
2973
|
+
const room = result.room;
|
|
2974
|
+
if (!room) {
|
|
2975
|
+
keyValue("Room", `no record room (format ${result.documentFormat ?? 1})`);
|
|
2976
|
+
return;
|
|
2977
|
+
}
|
|
2978
|
+
keyValue("Log number", room.lsn);
|
|
2979
|
+
keyValue("Epochs", `current ${room.epochs.current}, next ${room.epochs.next}, high ${room.epochs.high}`);
|
|
2980
|
+
keyValue("Epoch floor", room.epochs.floor);
|
|
2981
|
+
keyValue("Cursor floor", room.cursorFloor);
|
|
2982
|
+
keyValue("Entries since rotation", room.epochs.entriesSinceRotation);
|
|
2983
|
+
keyValue("Rotated", room.epochs.rotatedAt ? formatDate(room.epochs.rotatedAt) : "—");
|
|
2984
|
+
keyValue("Writers", `${room.writers.count} (${room.writers.connected} connected)`);
|
|
2985
|
+
keyValue("Subscribers", room.subscribers);
|
|
2986
|
+
keyValue("Held entries", room.held.entries);
|
|
2987
|
+
const load = room.load;
|
|
2988
|
+
keyValue("Load", load
|
|
2989
|
+
? `${load.sessionId} ${load.state} (${load.mode}, epoch reserved at ${load.reserveEpoch}` +
|
|
2990
|
+
(load.epoch !== null ? `: epoch ${load.epoch} from log number ${load.k}` : "") +
|
|
2991
|
+
`), ${load.chunksApplied}/${load.chunks} chunks, ${load.rowsApplied} rows applied, ` +
|
|
2992
|
+
`${load.refusedRows} refused, ${load.touched} keys touched`
|
|
2993
|
+
: "none");
|
|
2994
|
+
const loads = Array.isArray(room.loads) ? room.loads : [];
|
|
2995
|
+
keyValue("Loads", loads.length
|
|
2996
|
+
? loads
|
|
2997
|
+
.map((l) => `${l.sessionId} (${l.mode}, log numbers ${l.firstLsn ?? "—"}–${l.lastLsn ?? "—"}, ${l.rows} rows, ${l.touched} keys touched)`)
|
|
2998
|
+
.join("; ")
|
|
2999
|
+
: "none kept");
|
|
3000
|
+
const segmentLine = (s) => `${s.segmentId} ${s.state} (log number ${s.lsn}` +
|
|
3001
|
+
(s.completedLsn !== null && s.completedLsn !== undefined ? `, completed at ${s.completedLsn}` : "") +
|
|
3002
|
+
`, ${s.rows} rows, ${s.chunks} chunks)`;
|
|
3003
|
+
keyValue("Segment", room.segment ? segmentLine(room.segment) : "none");
|
|
3004
|
+
keyValue("Segment export", room.segmentExport ? segmentLine(room.segmentExport) : "none");
|
|
3005
|
+
keyValue("Last checkpoint", room.checkpoint
|
|
3006
|
+
? `${formatDate(room.checkpoint.at)} (${room.checkpoint.durationMs} ms, ` +
|
|
3007
|
+
`${room.checkpoint.pruned} pruned, ${room.checkpoint.dropped} dropped, ` +
|
|
3008
|
+
`${room.checkpoint.writersDropped} writers dropped)`
|
|
3009
|
+
: "none yet");
|
|
3010
|
+
keyValue("Next alarm", room.nextAlarmAt ? formatDate(room.nextAlarmAt) : "—");
|
|
3011
|
+
keyValue("Storage (bytes)", room.storage?.databaseSize ?? "—");
|
|
3012
|
+
}
|
|
3013
|
+
catch (err) {
|
|
3014
|
+
error(describeDocumentFailure(err));
|
|
3015
|
+
process.exit(1);
|
|
3013
3016
|
}
|
|
3014
3017
|
}
|
|
3015
3018
|
/**
|
|
3016
|
-
*
|
|
3017
|
-
*
|
|
3018
|
-
*
|
|
3019
|
-
* arm, and the schema and records reads it is compared against take the
|
|
3020
|
-
* console arm or a read grant on the document — the documents surface grants
|
|
3021
|
-
* no app-role bypass on a read (#3279, D3279-002). The schema is read first,
|
|
3022
|
-
* so a caller holding only the first is refused before anything is exported.
|
|
3019
|
+
* `documents ingest` and `db3 ingest` (#4889): load a directory into a
|
|
3020
|
+
* document or a db3 through `routes`, confirming unless `-y`, then exit with
|
|
3021
|
+
* the load's code.
|
|
3023
3022
|
*/
|
|
3024
|
-
async function
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
//
|
|
3030
|
-
//
|
|
3031
|
-
//
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
3023
|
+
export async function ingestAction(routes, documentId, options) {
|
|
3024
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
3025
|
+
const client = new ApiClient();
|
|
3026
|
+
let plan = null;
|
|
3027
|
+
// The exit code is carried out of here rather than taken inside: a
|
|
3028
|
+
// `process.exit` does not unwind, so ending the process in the try block
|
|
3029
|
+
// would skip the cleanup in the finally and leave the compressed
|
|
3030
|
+
// artifact — up to the whole input, gzipped — in the OS temp directory
|
|
3031
|
+
// on every run, successful or not (finding 3435-FROZEN-05).
|
|
3032
|
+
const run = async () => {
|
|
3033
|
+
// A flag's value is checked before any request.
|
|
3034
|
+
const flags = parseIngestFlags({ reserveEpoch: options.reserveEpoch, mode: options.mode });
|
|
3035
|
+
// The preflight, in this order and before a byte is read from disk:
|
|
3036
|
+
// the schema read says whether this caller may see the document's
|
|
3037
|
+
// declarations at all and hands over the ones the lines are checked
|
|
3038
|
+
// against, and the session reader says whether this is a large
|
|
3039
|
+
// document. Both are READS on the arm the ingest routes admit, so a
|
|
3040
|
+
// caller who would be refused halfway is refused at once, and neither
|
|
3041
|
+
// opens a session.
|
|
3042
|
+
const introspection = await routes.schema(client, resolvedAppId, documentId);
|
|
3043
|
+
await routes.listIngests(client, resolvedAppId, documentId, { limit: 1 });
|
|
3044
|
+
const schema = ingestSchemaFromIntrospection(introspection);
|
|
3045
|
+
// A record-log (format 3) document takes models and fields it does
|
|
3046
|
+
// not declare, as its writes do (#4310); its schema read says so.
|
|
3047
|
+
const recordLog = Number(introspection?.documentFormat) === 3;
|
|
3048
|
+
const input = discoverIngestInput(options.input, documentId);
|
|
3049
|
+
for (const ignored of input.ignored) {
|
|
3050
|
+
info(`Ignoring ${ignored}: a bulk load reads the snapshot only.`);
|
|
3051
|
+
}
|
|
3052
|
+
const body = ingestCreateBody(flags, input.sources.map((source) => source.model));
|
|
3053
|
+
// #3598 — reading, checking, sorting, cutting and gzipping is this
|
|
3054
|
+
// side's biggest phase on a large input, and it used to be invisible
|
|
3055
|
+
// inside one wall-clock number that also covered the platform's work.
|
|
3056
|
+
const buildStartedAt = Date.now();
|
|
3057
|
+
plan = await buildIngestArtifact({ input, schema, open: recordLog });
|
|
3058
|
+
const buildMs = Date.now() - buildStartedAt;
|
|
3059
|
+
info(`Built artifact in ${(buildMs / 1000).toFixed(1)}s.`);
|
|
3060
|
+
for (const line of summariseIngestPlan(plan))
|
|
3061
|
+
info(line);
|
|
3062
|
+
// A format-2 export's row that cannot be one record (#4638): left
|
|
3063
|
+
// out, and named, before anything is uploaded.
|
|
3064
|
+
for (const row of plan.refused) {
|
|
3065
|
+
warn(`Refused before upload: ${row.where}: ${row.reason}`);
|
|
3066
|
+
}
|
|
3067
|
+
assertIngestRefreshKeepsRefused(flags, plan.refused);
|
|
3068
|
+
assertIngestConfirmable({
|
|
3069
|
+
yes: Boolean(options.yes),
|
|
3070
|
+
isTTY: Boolean(process.stdin.isTTY),
|
|
3071
|
+
});
|
|
3072
|
+
if (!options.yes) {
|
|
3073
|
+
const confirmed = await confirmPrompt(`Bulk-load ${plan.rows} records into ${documentId}? ` +
|
|
3074
|
+
`This replaces records in a live ${routes.noun} and cannot be undone.`);
|
|
3075
|
+
if (!confirmed) {
|
|
3076
|
+
info("Nothing was uploaded.");
|
|
3077
|
+
return 1;
|
|
3078
|
+
}
|
|
3079
|
+
}
|
|
3080
|
+
// Ctrl-C is a decision the driver makes differently depending on
|
|
3081
|
+
// where it lands, so it is handed the signal rather than killing the
|
|
3082
|
+
// process: mid-upload the session is aborted, mid-poll it is left
|
|
3083
|
+
// running, because by then the document is already being changed.
|
|
3084
|
+
const interrupt = new AbortController();
|
|
3085
|
+
const onSignal = () => interrupt.abort();
|
|
3086
|
+
process.on("SIGINT", onSignal);
|
|
3087
|
+
let outcome;
|
|
3088
|
+
try {
|
|
3089
|
+
outcome = await driveDocumentIngest({
|
|
3090
|
+
client: {
|
|
3091
|
+
createIngest: (id) => routes.createIngest(client, resolvedAppId, id, body),
|
|
3092
|
+
uploadIngestChunk: (id, sessionId, chunk) => routes.uploadIngestChunk(client, resolvedAppId, id, sessionId, chunk),
|
|
3093
|
+
commitIngest: (id, sessionId) => routes.commitIngest(client, resolvedAppId, id, sessionId),
|
|
3094
|
+
abortIngest: (id, sessionId) => routes.abortIngest(client, resolvedAppId, id, sessionId),
|
|
3095
|
+
getIngest: (id, sessionId) => routes.getIngest(client, resolvedAppId, id, sessionId),
|
|
3096
|
+
},
|
|
3097
|
+
documentId,
|
|
3098
|
+
plan,
|
|
3099
|
+
wait: options.wait !== false,
|
|
3100
|
+
timeoutMs: options.timeout ? Number(options.timeout) * 1000 : null,
|
|
3101
|
+
signal: interrupt.signal,
|
|
3102
|
+
onProgress: (line) => info(line),
|
|
3103
|
+
});
|
|
3104
|
+
}
|
|
3105
|
+
finally {
|
|
3106
|
+
process.off("SIGINT", onSignal);
|
|
3107
|
+
}
|
|
3108
|
+
if (outcome.message)
|
|
3109
|
+
error(outcome.message);
|
|
3110
|
+
// #3598 — the two sides, side by side. The session's own `timings`
|
|
3111
|
+
// say where the platform's share went; `client` says what this
|
|
3112
|
+
// process spent before and around it, so a slow load can be blamed on
|
|
3113
|
+
// the side that was actually slow.
|
|
3114
|
+
const clientTimings = { buildMs, ...outcome.timings };
|
|
3115
|
+
if (options.json) {
|
|
3116
|
+
json({
|
|
3117
|
+
...(outcome.session ?? { sessionId: outcome.sessionId }),
|
|
3118
|
+
client: clientTimings,
|
|
3119
|
+
refusedBeforeUpload: plan.refused,
|
|
3120
|
+
});
|
|
3121
|
+
}
|
|
3122
|
+
else {
|
|
3123
|
+
if (outcome.session) {
|
|
3124
|
+
printIngestSession(outcome.session);
|
|
3125
|
+
}
|
|
3126
|
+
else if (outcome.sessionId) {
|
|
3127
|
+
printResult("Session", outcome.sessionId);
|
|
3128
|
+
}
|
|
3129
|
+
for (const [label, value] of ingestClientRows(clientTimings)) {
|
|
3130
|
+
printResult(label, value);
|
|
3131
|
+
}
|
|
3132
|
+
}
|
|
3133
|
+
return outcome.exitCode;
|
|
3134
|
+
};
|
|
3135
|
+
let code = 1;
|
|
3136
|
+
try {
|
|
3137
|
+
code = await run();
|
|
3138
|
+
}
|
|
3139
|
+
catch (err) {
|
|
3140
|
+
error(describeDocumentFailure(err));
|
|
3141
|
+
code = 1;
|
|
3142
|
+
}
|
|
3143
|
+
finally {
|
|
3144
|
+
await plan?.cleanup().catch(() => undefined);
|
|
3145
|
+
}
|
|
3146
|
+
// The result has to reach the pipe before the process ends: stdout to a
|
|
3147
|
+
// pipe is asynchronous and `process.exit` does not wait for it.
|
|
3148
|
+
await flushOutput();
|
|
3149
|
+
process.exit(code);
|
|
3150
|
+
}
|
|
3151
|
+
/** `documents ingests list` and `db3 ingests list` (#4889). */
|
|
3152
|
+
export async function listIngestsAction(routes, documentId, options) {
|
|
3153
|
+
const limit = parsePageLimit(options.limit);
|
|
3154
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
3155
|
+
const client = new ApiClient();
|
|
3156
|
+
try {
|
|
3157
|
+
const page = normalizeCliListEnvelope(await routes.listIngests(client, resolvedAppId, documentId, {
|
|
3158
|
+
limit,
|
|
3159
|
+
...(options.cursor ? { cursor: options.cursor } : {}),
|
|
3160
|
+
}));
|
|
3161
|
+
if (options.json) {
|
|
3162
|
+
json(page);
|
|
3163
|
+
return;
|
|
3164
|
+
}
|
|
3165
|
+
if (page.items.length === 0) {
|
|
3166
|
+
info("No ingest sessions yet.");
|
|
3167
|
+
return;
|
|
3168
|
+
}
|
|
3169
|
+
console.log(formatTable(page.items, [
|
|
3170
|
+
{ header: "SESSION", key: "sessionId" },
|
|
3171
|
+
{ header: "STATE", key: "state" },
|
|
3172
|
+
{ header: "CREATED", key: "createdAt", format: formatDate },
|
|
3173
|
+
{ header: "CHUNKS", key: "chunks" },
|
|
3174
|
+
{ header: "ROWS", key: "rows" },
|
|
3175
|
+
{
|
|
3176
|
+
header: "COMPLETED",
|
|
3177
|
+
key: "completedAt",
|
|
3178
|
+
format: (at) => (at ? formatDate(at) : "—"),
|
|
3179
|
+
},
|
|
3180
|
+
]));
|
|
3181
|
+
if (page.hasMore && page.nextCursor) {
|
|
3182
|
+
info(`More sessions: --cursor ${page.nextCursor}`);
|
|
3183
|
+
}
|
|
3184
|
+
}
|
|
3185
|
+
catch (err) {
|
|
3186
|
+
error(describeDocumentFailure(err));
|
|
3187
|
+
process.exit(1);
|
|
3188
|
+
}
|
|
3189
|
+
}
|
|
3190
|
+
/** `documents ingests get` and `db3 ingests get` (#4889). */
|
|
3191
|
+
export async function getIngestAction(routes, documentId, sessionId, options) {
|
|
3192
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
3193
|
+
const client = new ApiClient();
|
|
3194
|
+
try {
|
|
3195
|
+
const session = await routes.getIngest(client, resolvedAppId, documentId, sessionId);
|
|
3196
|
+
if (options.json) {
|
|
3197
|
+
json(session);
|
|
3198
|
+
return;
|
|
3199
|
+
}
|
|
3200
|
+
printIngestSession(session);
|
|
3201
|
+
}
|
|
3202
|
+
catch (err) {
|
|
3203
|
+
error(describeDocumentFailure(err));
|
|
3204
|
+
process.exit(1);
|
|
3205
|
+
}
|
|
3206
|
+
}
|
|
3207
|
+
/** `documents ingests abort` and `db3 ingests abort` (#4889). */
|
|
3208
|
+
export async function abortIngestAction(routes, documentId, sessionId, options) {
|
|
3209
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
3210
|
+
if (!options.yes) {
|
|
3211
|
+
if (!process.stdin.isTTY) {
|
|
3212
|
+
error("Aborting a bulk load discards what it uploaded. " +
|
|
3213
|
+
"Re-run with -y to confirm it without a prompt.");
|
|
3214
|
+
process.exit(1);
|
|
3215
|
+
}
|
|
3216
|
+
const confirmed = await confirmPrompt(`Abort bulk load ${sessionId} on ${documentId}? What it uploaded is ` +
|
|
3217
|
+
"discarded; rows a committed load already applied stay.");
|
|
3218
|
+
if (!confirmed) {
|
|
3219
|
+
info("Cancelled.");
|
|
3220
|
+
return;
|
|
3221
|
+
}
|
|
3222
|
+
}
|
|
3223
|
+
const client = new ApiClient();
|
|
3224
|
+
try {
|
|
3225
|
+
const session = await routes.abortIngest(client, resolvedAppId, documentId, sessionId);
|
|
3226
|
+
if (options.json) {
|
|
3227
|
+
json(session);
|
|
3228
|
+
return;
|
|
3229
|
+
}
|
|
3230
|
+
printIngestSession(session);
|
|
3231
|
+
}
|
|
3232
|
+
catch (err) {
|
|
3233
|
+
error(describeDocumentFailure(err));
|
|
3234
|
+
process.exit(1);
|
|
3235
|
+
}
|
|
3236
|
+
}
|
|
3237
|
+
/** `documents snapshots list` and `db3 snapshots list` (#4889). */
|
|
3238
|
+
export async function listSnapshotsAction(routes, documentId, options) {
|
|
3239
|
+
const limit = parsePageLimit(options.limit);
|
|
3240
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
3241
|
+
const client = new ApiClient();
|
|
3242
|
+
try {
|
|
3243
|
+
const page = normalizeCliListEnvelope(await routes.listSnapshots(client, resolvedAppId, documentId, {
|
|
3244
|
+
limit,
|
|
3245
|
+
...(options.cursor ? { cursor: options.cursor } : {}),
|
|
3246
|
+
}));
|
|
3247
|
+
if (options.json) {
|
|
3248
|
+
json(page);
|
|
3249
|
+
return;
|
|
3250
|
+
}
|
|
3251
|
+
if (page.items.length === 0) {
|
|
3252
|
+
info("No snapshot builds yet.");
|
|
3253
|
+
return;
|
|
3254
|
+
}
|
|
3255
|
+
// A record-log (format 3) document's are its segments (#4310).
|
|
3256
|
+
if (page.items.some((item) => typeof item?.segmentId === "string")) {
|
|
3257
|
+
console.log(formatTable(page.items, [
|
|
3258
|
+
{ header: "SEGMENT", key: "segmentId" },
|
|
3259
|
+
{ header: "LOG NUMBER", key: "lsn" },
|
|
3260
|
+
{ header: "COMPLETED AT LOG NUMBER", key: "completedLsn", format: (v) => (v === null || v === undefined ? "-" : String(v)) },
|
|
3261
|
+
{ header: "STATE", key: "state" },
|
|
3262
|
+
{ header: "ROWS", key: "rows" },
|
|
3263
|
+
{ header: "CHUNKS", key: "chunks" },
|
|
3264
|
+
{ header: "STARTED", key: "startedAt", format: formatDate },
|
|
3265
|
+
{ header: "COMPLETED", key: "completedAt", format: formatDate },
|
|
3266
|
+
]));
|
|
3267
|
+
return;
|
|
3268
|
+
}
|
|
3269
|
+
console.log(formatTable(page.items, [
|
|
3270
|
+
{ header: "BUILD", key: "buildId" },
|
|
3271
|
+
{ header: "EPOCH", key: "epoch" },
|
|
3272
|
+
{ header: "STATE", key: "state" },
|
|
3273
|
+
{ header: "SOURCE", key: "source" },
|
|
3274
|
+
{
|
|
3275
|
+
header: "VERIFICATION",
|
|
3276
|
+
key: "verification",
|
|
3277
|
+
format: (verification) => verification
|
|
3278
|
+
? verification.code
|
|
3279
|
+
? `${verification.state} (${verification.code})`
|
|
3280
|
+
: verification.state
|
|
3281
|
+
: "-",
|
|
3282
|
+
},
|
|
3283
|
+
{ header: "ROWS", key: "rows" },
|
|
3284
|
+
{ header: "CHUNKS", key: "chunks" },
|
|
3285
|
+
{ header: "STARTED", key: "startedAt", format: formatDate },
|
|
3286
|
+
{ header: "COMPLETED", key: "completedAt", format: formatDate },
|
|
3287
|
+
]));
|
|
3288
|
+
if (page.hasMore && page.nextCursor) {
|
|
3289
|
+
info(`More builds: --cursor ${page.nextCursor}`);
|
|
3290
|
+
}
|
|
3291
|
+
}
|
|
3292
|
+
catch (err) {
|
|
3293
|
+
error(describeDocumentFailure(err));
|
|
3294
|
+
process.exit(1);
|
|
3295
|
+
}
|
|
3296
|
+
}
|
|
3297
|
+
/** `documents snapshots get` and `db3 snapshots get` (#4889). */
|
|
3298
|
+
export async function getSnapshotAction(routes, documentId, buildId, options) {
|
|
3299
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
3300
|
+
const client = new ApiClient();
|
|
3301
|
+
try {
|
|
3302
|
+
const build = await routes.getSnapshot(client, resolvedAppId, documentId, buildId);
|
|
3303
|
+
if (options.json) {
|
|
3304
|
+
json(build);
|
|
3305
|
+
return;
|
|
3306
|
+
}
|
|
3307
|
+
// The rows are composed by a pure function (#3435), so what a
|
|
3308
|
+
// terminal shows is checkable without spawning one; the command still
|
|
3309
|
+
// owns the printing. A record-log document's build is a segment (#4310).
|
|
3310
|
+
const rows = typeof build?.segmentId === "string" ? segmentRows(build) : snapshotBuildRows(build);
|
|
3311
|
+
for (const [label, value] of rows) {
|
|
3312
|
+
printResult(label, value);
|
|
3313
|
+
}
|
|
3314
|
+
}
|
|
3315
|
+
catch (err) {
|
|
3316
|
+
error(describeDocumentFailure(err));
|
|
3317
|
+
process.exit(1);
|
|
3318
|
+
}
|
|
3319
|
+
}
|
|
3320
|
+
/** `documents snapshots build` and `db3 snapshots build` (#4889). */
|
|
3321
|
+
export async function buildSnapshotAction(routes, documentId, options) {
|
|
3322
|
+
const resolvedAppId = resolveAppId(undefined, options);
|
|
3323
|
+
const client = new ApiClient();
|
|
3324
|
+
const run = async () => {
|
|
3325
|
+
// Ctrl-C while polling leaves the build running — by then the room is
|
|
3326
|
+
// already building it and giving up on watching is not giving up on
|
|
3327
|
+
// the build — so the signal is handed to the driver rather than
|
|
3328
|
+
// killing the process.
|
|
3329
|
+
const interrupt = new AbortController();
|
|
3330
|
+
const onSignal = () => interrupt.abort();
|
|
3331
|
+
process.on("SIGINT", onSignal);
|
|
3332
|
+
let outcome;
|
|
3333
|
+
try {
|
|
3334
|
+
outcome = await driveSnapshotBuild({
|
|
3335
|
+
client: {
|
|
3336
|
+
requestSnapshot: (id) => routes.requestSnapshot(client, resolvedAppId, id),
|
|
3337
|
+
getSnapshot: (id, buildId) => routes.getSnapshot(client, resolvedAppId, id, buildId),
|
|
3338
|
+
},
|
|
3339
|
+
documentId,
|
|
3340
|
+
wait: Boolean(options.wait),
|
|
3341
|
+
timeoutMs: options.timeout ? Number(options.timeout) * 1000 : null,
|
|
3342
|
+
signal: interrupt.signal,
|
|
3343
|
+
onProgress: (line) => info(line),
|
|
3344
|
+
});
|
|
3345
|
+
}
|
|
3346
|
+
finally {
|
|
3347
|
+
process.off("SIGINT", onSignal);
|
|
3348
|
+
}
|
|
3349
|
+
if (outcome.message)
|
|
3350
|
+
error(outcome.message);
|
|
3351
|
+
if (options.json) {
|
|
3352
|
+
// The route's answer as received, with the build the wait settled
|
|
3353
|
+
// on beside it when there was one — so a script branches on the
|
|
3354
|
+
// verdict rather than on an exit code alone.
|
|
3355
|
+
json(outcome.build ? { ...outcome.response, build: outcome.build } : outcome.response);
|
|
3356
|
+
return outcome.exitCode;
|
|
3357
|
+
}
|
|
3358
|
+
if (typeof outcome.response?.segmentId === "string") {
|
|
3359
|
+
// A record-log document's segment export (#4310).
|
|
3360
|
+
if (outcome.build) {
|
|
3361
|
+
for (const [label, value] of segmentRows(outcome.build))
|
|
3362
|
+
printResult(label, value);
|
|
3363
|
+
}
|
|
3364
|
+
else {
|
|
3365
|
+
printResult("Segment", outcome.response.segmentId);
|
|
3366
|
+
printResult("State", outcome.response.state);
|
|
3367
|
+
printResult("Log number", outcome.response.lsn);
|
|
3368
|
+
}
|
|
3369
|
+
return outcome.exitCode;
|
|
3370
|
+
}
|
|
3371
|
+
printResult("Build", outcome.response?.buildId);
|
|
3372
|
+
return outcome.exitCode;
|
|
3373
|
+
};
|
|
3374
|
+
let code = 1;
|
|
3375
|
+
try {
|
|
3376
|
+
code = await run();
|
|
3377
|
+
}
|
|
3378
|
+
catch (err) {
|
|
3379
|
+
// The server's own sentence, plus the code a script branches on
|
|
3380
|
+
// (#3403) — a refused loop that only says "too young to seal" leaves
|
|
3381
|
+
// a caller parsing prose to tell it from a permission refusal.
|
|
3382
|
+
error(err.code ? `${err.message} (${err.code})` : err.message);
|
|
3383
|
+
const retryAfterMs = err.details?.retryAfterMs;
|
|
3384
|
+
if (typeof retryAfterMs === "number") {
|
|
3385
|
+
error(`Ask again in ${Math.ceil(retryAfterMs / 1000)}s.`);
|
|
3386
|
+
}
|
|
3387
|
+
code = 1;
|
|
3388
|
+
}
|
|
3389
|
+
// The result has to reach the pipe before the process ends: stdout to a
|
|
3390
|
+
// pipe is asynchronous and `process.exit` does not wait for it.
|
|
3391
|
+
await flushOutput();
|
|
3392
|
+
process.exit(code);
|
|
3043
3393
|
}
|
|
3044
3394
|
//# sourceMappingURL=documents.js.map
|