wizards 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +110 -0
- package/README.md +170 -0
- package/apps/runtime/dist/agents/architect.js +453 -0
- package/apps/runtime/dist/agents/project-assistant.js +354 -0
- package/apps/runtime/dist/agents/subscription.js +206 -0
- package/apps/runtime/dist/agents/thought.js +21 -0
- package/apps/runtime/dist/agents/website.js +197 -0
- package/apps/runtime/dist/app.js +153 -0
- package/apps/runtime/dist/auth/account.js +121 -0
- package/apps/runtime/dist/auth/index.js +329 -0
- package/apps/runtime/dist/auth/keys.js +55 -0
- package/apps/runtime/dist/auth/local-ticket.js +23 -0
- package/apps/runtime/dist/authoring/example.js +73 -0
- package/apps/runtime/dist/authoring/guide.js +259 -0
- package/apps/runtime/dist/authoring/ops.js +117 -0
- package/apps/runtime/dist/cli/autostart.js +134 -0
- package/apps/runtime/dist/cli/clients.js +44 -0
- package/apps/runtime/dist/cli/environment.js +81 -0
- package/apps/runtime/dist/cli/exec.js +45 -0
- package/apps/runtime/dist/cli/home.js +74 -0
- package/apps/runtime/dist/cli/machine.js +145 -0
- package/apps/runtime/dist/cli/main.js +125 -0
- package/apps/runtime/dist/cli/setup.js +273 -0
- package/apps/runtime/dist/cli/start.js +190 -0
- package/apps/runtime/dist/cli/status.js +120 -0
- package/apps/runtime/dist/cli/ui.js +12 -0
- package/apps/runtime/dist/cli/update.js +58 -0
- package/apps/runtime/dist/connectors/builtin.js +70 -0
- package/apps/runtime/dist/connectors/external.js +400 -0
- package/apps/runtime/dist/connectors/gmail.js +107 -0
- package/apps/runtime/dist/connectors/imap.js +217 -0
- package/apps/runtime/dist/connectors/index.js +266 -0
- package/apps/runtime/dist/connectors/mail.js +72 -0
- package/apps/runtime/dist/connectors/outlook.js +90 -0
- package/apps/runtime/dist/credits/credits.js +103 -0
- package/apps/runtime/dist/credits/estimate.js +132 -0
- package/apps/runtime/dist/db/adopt.js +149 -0
- package/apps/runtime/dist/db/client.js +146 -0
- package/apps/runtime/dist/db/control-schema.js +70 -0
- package/apps/runtime/dist/db/migrations/control/0000_last_absorbing_man.sql +47 -0
- package/apps/runtime/dist/db/migrations/control/0001_gorgeous_nicolaos.sql +41 -0
- package/apps/runtime/dist/db/migrations/control/0002_secret_penance.sql +1 -0
- package/apps/runtime/dist/db/migrations/control/0003_nasty_mordo.sql +3 -0
- package/apps/runtime/dist/db/migrations/control/0004_breezy_eddie_brock.sql +6 -0
- package/apps/runtime/dist/db/migrations/control/meta/0000_snapshot.json +311 -0
- package/apps/runtime/dist/db/migrations/control/meta/0001_snapshot.json +599 -0
- package/apps/runtime/dist/db/migrations/control/meta/0002_snapshot.json +607 -0
- package/apps/runtime/dist/db/migrations/control/meta/0003_snapshot.json +311 -0
- package/apps/runtime/dist/db/migrations/control/meta/0004_snapshot.json +350 -0
- package/apps/runtime/dist/db/migrations/control/meta/_journal.json +41 -0
- package/apps/runtime/dist/db/migrations/tenant/0000_eminent_freak.sql +188 -0
- package/apps/runtime/dist/db/migrations/tenant/0001_salty_master_chief.sql +1 -0
- package/apps/runtime/dist/db/migrations/tenant/0002_cloudy_firebird.sql +1 -0
- package/apps/runtime/dist/db/migrations/tenant/0003_brown_warbound.sql +52 -0
- package/apps/runtime/dist/db/migrations/tenant/0004_romantic_nocturne.sql +6 -0
- package/apps/runtime/dist/db/migrations/tenant/meta/0000_snapshot.json +1323 -0
- package/apps/runtime/dist/db/migrations/tenant/meta/0001_snapshot.json +1330 -0
- package/apps/runtime/dist/db/migrations/tenant/meta/0002_snapshot.json +1337 -0
- package/apps/runtime/dist/db/migrations/tenant/meta/0003_snapshot.json +1605 -0
- package/apps/runtime/dist/db/migrations/tenant/meta/0004_snapshot.json +1645 -0
- package/apps/runtime/dist/db/migrations/tenant/meta/_journal.json +41 -0
- package/apps/runtime/dist/db/schema.js +278 -0
- package/apps/runtime/dist/documents/parse.js +230 -0
- package/apps/runtime/dist/engenty/connections-external/build-connector.js +193 -0
- package/apps/runtime/dist/engenty/connections-external/errors.js +11 -0
- package/apps/runtime/dist/engenty/connections-external/import-service.js +270 -0
- package/apps/runtime/dist/engenty/connections-external/importer/classify.js +34 -0
- package/apps/runtime/dist/engenty/connections-external/importer/map-auth.js +185 -0
- package/apps/runtime/dist/engenty/connections-external/importer/normalize-mcp.js +63 -0
- package/apps/runtime/dist/engenty/connections-external/importer/normalize-openapi.js +130 -0
- package/apps/runtime/dist/engenty/connections-external/invoke/http-invoker.js +143 -0
- package/apps/runtime/dist/engenty/connections-external/invoke/mcp-client.js +143 -0
- package/apps/runtime/dist/engenty/connections-external/net/guarded-fetch.js +119 -0
- package/apps/runtime/dist/engenty/connections-external/oauth-dcr.js +88 -0
- package/apps/runtime/dist/engenty/connections-external/registry-client.js +336 -0
- package/apps/runtime/dist/engenty/connections-external/registry-source.js +82 -0
- package/apps/runtime/dist/engenty/connections-external/types.js +7 -0
- package/apps/runtime/dist/engenty/connections-github/connector.js +209 -0
- package/apps/runtime/dist/engenty/connections-google/connectors/calendar.js +359 -0
- package/apps/runtime/dist/engenty/connections-google/connectors/contacts.js +94 -0
- package/apps/runtime/dist/engenty/connections-google/connectors/drive.js +290 -0
- package/apps/runtime/dist/engenty/connections-google/connectors/gmail.js +567 -0
- package/apps/runtime/dist/engenty/connections-google/definitions.js +11 -0
- package/apps/runtime/dist/engenty/connections-google/shared.js +73 -0
- package/apps/runtime/dist/engenty/connections-hubspot/connector.js +140 -0
- package/apps/runtime/dist/engenty/connections-microsoft/action.js +17 -0
- package/apps/runtime/dist/engenty/connections-microsoft/graph.js +114 -0
- package/apps/runtime/dist/engenty/connections-microsoft/onedrive.js +286 -0
- package/apps/runtime/dist/engenty/connections-microsoft/outlook.js +464 -0
- package/apps/runtime/dist/engenty/connections-s3/s3.js +328 -0
- package/apps/runtime/dist/engenty/connections-sdk/files-capability.js +86 -0
- package/apps/runtime/dist/engenty/connections-sdk/oauth2.js +150 -0
- package/apps/runtime/dist/engenty/connections-sdk/registry.js +89 -0
- package/apps/runtime/dist/engenty/connections-sdk/storage-capability.js +84 -0
- package/apps/runtime/dist/engenty/connections-sdk/types.js +30 -0
- package/apps/runtime/dist/engenty/connections-slack/connector.js +360 -0
- package/apps/runtime/dist/engenty/csv-import/csv-matrix.js +178 -0
- package/apps/runtime/dist/engenty/csv-import/import-sources.js +87 -0
- package/apps/runtime/dist/engenty/csv-import/parse-csv.js +129 -0
- package/apps/runtime/dist/engenty/csv-import/serialize-csv.js +20 -0
- package/apps/runtime/dist/engenty/csv-import/types.js +1 -0
- package/apps/runtime/dist/engenty/csv-import/xlsx-workbook.js +159 -0
- package/apps/runtime/dist/engenty/doc-converter/interface.js +9 -0
- package/apps/runtime/dist/engenty/doc-converter/page-break.js +92 -0
- package/apps/runtime/dist/engenty/doc-converter/providers/local/index.js +72 -0
- package/apps/runtime/dist/engenty/document-scanner/schemas/classifier.js +30 -0
- package/apps/runtime/dist/engenty/document-scanner/schemas/index.js +4 -0
- package/apps/runtime/dist/engenty/document-scanner/schemas/invoice.js +73 -0
- package/apps/runtime/dist/engenty/document-scanner/schemas/receipt.js +63 -0
- package/apps/runtime/dist/engenty/document-scanner/schemas/shared.js +40 -0
- package/apps/runtime/dist/engenty/search-index/chunks.js +164 -0
- package/apps/runtime/dist/engenty/search-index/contracts.js +9 -0
- package/apps/runtime/dist/engenty/shims/connections-sdk.js +39 -0
- package/apps/runtime/dist/engenty/shims/plugin-sdk.js +1 -0
- package/apps/runtime/dist/engenty/shims/web-ingest.js +3 -0
- package/apps/runtime/dist/engenty/web-ingest/ssrf.js +277 -0
- package/apps/runtime/dist/engine/asks.js +60 -0
- package/apps/runtime/dist/engine/events.js +46 -0
- package/apps/runtime/dist/engine/input.js +170 -0
- package/apps/runtime/dist/engine/prepare.js +51 -0
- package/apps/runtime/dist/engine/requirements.js +124 -0
- package/apps/runtime/dist/engine/resources.js +89 -0
- package/apps/runtime/dist/engine/runner.js +560 -0
- package/apps/runtime/dist/engine/steps.js +598 -0
- package/apps/runtime/dist/engine/template.js +183 -0
- package/apps/runtime/dist/engine/types.js +3 -0
- package/apps/runtime/dist/env.js +224 -0
- package/apps/runtime/dist/files/blobs.js +22 -0
- package/apps/runtime/dist/files/objects.js +86 -0
- package/apps/runtime/dist/files/storage.js +95 -0
- package/apps/runtime/dist/geocode.js +58 -0
- package/apps/runtime/dist/harness/claude.js +138 -0
- package/apps/runtime/dist/harness/codex.js +228 -0
- package/apps/runtime/dist/harness/cursor.js +93 -0
- package/apps/runtime/dist/harness/env.js +124 -0
- package/apps/runtime/dist/harness/gemini.js +126 -0
- package/apps/runtime/dist/harness/index.js +99 -0
- package/apps/runtime/dist/harness/model.js +267 -0
- package/apps/runtime/dist/harness/prompt.js +121 -0
- package/apps/runtime/dist/harness/terminal.js +193 -0
- package/apps/runtime/dist/harness/types.js +7 -0
- package/apps/runtime/dist/index.js +76 -0
- package/apps/runtime/dist/limits.js +57 -0
- package/apps/runtime/dist/link-preview.js +59 -0
- package/apps/runtime/dist/manage.js +151 -0
- package/apps/runtime/dist/mcp/auth.js +102 -0
- package/apps/runtime/dist/mcp/bridge.js +59 -0
- package/apps/runtime/dist/mcp/handler.js +42 -0
- package/apps/runtime/dist/mcp/scopes.js +5 -0
- package/apps/runtime/dist/mcp/tools.js +401 -0
- package/apps/runtime/dist/media/ffmpeg.js +121 -0
- package/apps/runtime/dist/media/generate.js +120 -0
- package/apps/runtime/dist/media/marking.js +121 -0
- package/apps/runtime/dist/media/transcribe.js +30 -0
- package/apps/runtime/dist/model-errors.js +3 -0
- package/apps/runtime/dist/models.js +489 -0
- package/apps/runtime/dist/plugin.js +61 -0
- package/apps/runtime/dist/render/chromium.js +85 -0
- package/apps/runtime/dist/render/convert.js +161 -0
- package/apps/runtime/dist/render/downloads.js +277 -0
- package/apps/runtime/dist/render/guard.js +25 -0
- package/apps/runtime/dist/routes/api.js +74 -0
- package/apps/runtime/dist/routes/delivery.js +52 -0
- package/apps/runtime/dist/routes/internal.js +49 -0
- package/apps/runtime/dist/routes/marketplace.js +36 -0
- package/apps/runtime/dist/routes/runs.js +805 -0
- package/apps/runtime/dist/routes/studio.js +702 -0
- package/apps/runtime/dist/routes/wizard-stream.js +30 -0
- package/apps/runtime/dist/running.js +39 -0
- package/apps/runtime/dist/sandbox/agentos.js +191 -0
- package/apps/runtime/dist/sandbox/docker.js +77 -0
- package/apps/runtime/dist/sandbox/index.js +30 -0
- package/apps/runtime/dist/sandbox/types.js +1 -0
- package/apps/runtime/dist/secrets/crypto.js +28 -0
- package/apps/runtime/dist/secrets/signing.js +34 -0
- package/apps/runtime/dist/secrets/vault.js +59 -0
- package/apps/runtime/dist/services/architect.js +92 -0
- package/apps/runtime/dist/services/brand.js +20 -0
- package/apps/runtime/dist/services/cloud.js +52 -0
- package/apps/runtime/dist/services/draft-events.js +27 -0
- package/apps/runtime/dist/services/errors.js +24 -0
- package/apps/runtime/dist/services/files.js +129 -0
- package/apps/runtime/dist/services/links.js +16 -0
- package/apps/runtime/dist/services/marketplace.js +292 -0
- package/apps/runtime/dist/services/package.js +202 -0
- package/apps/runtime/dist/services/profile.js +23 -0
- package/apps/runtime/dist/services/project-files.js +266 -0
- package/apps/runtime/dist/services/project-index.js +109 -0
- package/apps/runtime/dist/services/projects.js +181 -0
- package/apps/runtime/dist/services/runs.js +224 -0
- package/apps/runtime/dist/services/shares.js +129 -0
- package/apps/runtime/dist/services/widgets.js +34 -0
- package/apps/runtime/dist/services/wizards.js +342 -0
- package/apps/runtime/dist/settings.js +16 -0
- package/apps/runtime/dist/store/index.js +279 -0
- package/apps/runtime/dist/tenants/control.js +106 -0
- package/apps/runtime/dist/tenants/tenant.js +26 -0
- package/apps/runtime/dist/tools/browser.js +380 -0
- package/apps/runtime/dist/tools/connector.js +204 -0
- package/apps/runtime/dist/tools/index.js +234 -0
- package/apps/runtime/dist/tools/mail.js +104 -0
- package/apps/runtime/dist/tools/net-guard.js +59 -0
- package/apps/runtime/dist/tools/project.js +53 -0
- package/apps/runtime/dist/tools/shared.js +42 -0
- package/apps/runtime/dist/tools/store.js +238 -0
- package/apps/runtime/dist/update.js +118 -0
- package/apps/runtime/dist/widgets/bundle.js +99 -0
- package/apps/runtime/dist/widgets/render.js +312 -0
- package/apps/runtime/dist/widgets/runtime.js +84 -0
- package/apps/runtime/dist/widgets/script.js +30 -0
- package/apps/runtime/dist/widgets/step.js +192 -0
- package/apps/web/dist/ai-labels/ai-black.svg +20 -0
- package/apps/web/dist/ai-labels/ai-generated-black.svg +31 -0
- package/apps/web/dist/ai-labels/ai-generated-white.svg +32 -0
- package/apps/web/dist/ai-labels/ai-modified-black.svg +30 -0
- package/apps/web/dist/ai-labels/ai-modified-white.svg +31 -0
- package/apps/web/dist/ai-labels/ai-white.svg +21 -0
- package/apps/web/dist/assets/ChangelogDialog-O00vEnbm.js +1 -0
- package/apps/web/dist/assets/ChatPanel-CH9ry0Hp.js +1 -0
- package/apps/web/dist/assets/CreditsDialog-DRzecwUw.js +1 -0
- package/apps/web/dist/assets/EditorPage-BRnBf02Q.js +2 -0
- package/apps/web/dist/assets/FlowPreview-DDOj8IAH.js +1 -0
- package/apps/web/dist/assets/HomePage-CAGTTG3N.js +1 -0
- package/apps/web/dist/assets/ImportWizard-0RRrAnX3.js +1 -0
- package/apps/web/dist/assets/LocalRuntime-BrP-ENHg.css +1 -0
- package/apps/web/dist/assets/LocalRuntime-C-TBYrzs.js +16 -0
- package/apps/web/dist/assets/NewWizardPage-kI90h3_9.js +2 -0
- package/apps/web/dist/assets/PublicRunner-CKRDeIVd.js +1 -0
- package/apps/web/dist/assets/RunnerView-Rc4H8wYH.js +2 -0
- package/apps/web/dist/assets/SettingsPage-CESMuZxq.js +2 -0
- package/apps/web/dist/assets/SetupPage-D1rrYU14.js +1 -0
- package/apps/web/dist/assets/SharePage-COTAmOgh.js +1 -0
- package/apps/web/dist/assets/api-BY28cjGY.js +10 -0
- package/apps/web/dist/assets/arrow-left-UpjOBfQp.js +1 -0
- package/apps/web/dist/assets/arrow-right-CucQycEy.js +1 -0
- package/apps/web/dist/assets/changelog-p3wsaDDS.js +1 -0
- package/apps/web/dist/assets/check-Bjx41xbB.js +1 -0
- package/apps/web/dist/assets/createLucideIcon-DBu2Ovxj.js +1 -0
- package/apps/web/dist/assets/definition-cy7uvY5L.js +55 -0
- package/apps/web/dist/assets/engenty-CWIFpgKq.js +1 -0
- package/apps/web/dist/assets/esm-D1OEhjTK.js +7 -0
- package/apps/web/dist/assets/external-link-CRg6DVdO.js +1 -0
- package/apps/web/dist/assets/fluffy-engenty-CJ3wCQgT.js +456 -0
- package/apps/web/dist/assets/index-CQ_q9iMO.css +1 -0
- package/apps/web/dist/assets/index-D0gxh90t.js +10 -0
- package/apps/web/dist/assets/jsx-runtime-Dk72oS4N.js +1 -0
- package/apps/web/dist/assets/meta-BbSdtTUa.js +1 -0
- package/apps/web/dist/assets/mic-CeqQ89Dj.js +1 -0
- package/apps/web/dist/assets/oss-credits-cPuZUFHD.js +1 -0
- package/apps/web/dist/assets/outputs-D6pxH6ve.js +69 -0
- package/apps/web/dist/assets/plus-HA210wXA.js +1 -0
- package/apps/web/dist/assets/preload-helper-CNEI7N0N.js +1 -0
- package/apps/web/dist/assets/qr-scanner-worker.min-CJaXo3yu.js +98 -0
- package/apps/web/dist/assets/qr-scanner.min-C_UI0gH-.js +1 -0
- package/apps/web/dist/assets/type-BzYgVWKL.js +1 -0
- package/apps/web/dist/assets/ui-Ddwl35A6.js +1 -0
- package/apps/web/dist/assets/upload-DsgLfQ3U.js +1 -0
- package/apps/web/dist/assets/video-Dk1fuTDl.js +1 -0
- package/apps/web/dist/assets/wand-sparkles-Bf97fs8T.js +1 -0
- package/apps/web/dist/embed.js +102 -0
- package/apps/web/dist/favicon.svg +1 -0
- package/apps/web/dist/icons/apple-touch-icon.png +0 -0
- package/apps/web/dist/icons/icon-192.png +0 -0
- package/apps/web/dist/icons/icon-512.png +0 -0
- package/apps/web/dist/icons/icon-maskable-512.png +0 -0
- package/apps/web/dist/index.html +46 -0
- package/apps/web/dist/install.sh +532 -0
- package/apps/web/dist/manifest.webmanifest +20 -0
- package/apps/web/dist/sw.js +15 -0
- package/bin/wizards.mjs +13 -0
- package/node_modules/@engenty-wizards/shared/dist/connectors.d.ts +77 -0
- package/node_modules/@engenty-wizards/shared/dist/connectors.js +2 -0
- package/node_modules/@engenty-wizards/shared/dist/definition.d.ts +1415 -0
- package/node_modules/@engenty-wizards/shared/dist/definition.js +762 -0
- package/node_modules/@engenty-wizards/shared/dist/engenty/data-tables/columns-format.d.ts +5 -0
- package/node_modules/@engenty-wizards/shared/dist/engenty/data-tables/columns-format.js +132 -0
- package/node_modules/@engenty-wizards/shared/dist/engenty/data-tables/columns-value.d.ts +11 -0
- package/node_modules/@engenty-wizards/shared/dist/engenty/data-tables/columns-value.js +209 -0
- package/node_modules/@engenty-wizards/shared/dist/engenty/data-tables/columns.d.ts +532 -0
- package/node_modules/@engenty-wizards/shared/dist/engenty/data-tables/columns.js +216 -0
- package/node_modules/@engenty-wizards/shared/dist/engenty/data-tables/index.d.ts +3 -0
- package/node_modules/@engenty-wizards/shared/dist/engenty/data-tables/index.js +3 -0
- package/node_modules/@engenty-wizards/shared/dist/engenty/search-index/catalog-lexical.d.ts +14 -0
- package/node_modules/@engenty-wizards/shared/dist/engenty/search-index/catalog-lexical.js +166 -0
- package/node_modules/@engenty-wizards/shared/dist/engenty/search-index/catalog-rank.d.ts +33 -0
- package/node_modules/@engenty-wizards/shared/dist/engenty/search-index/catalog-rank.js +99 -0
- package/node_modules/@engenty-wizards/shared/dist/marketplace-entry.d.ts +35 -0
- package/node_modules/@engenty-wizards/shared/dist/marketplace-entry.js +86 -0
- package/node_modules/@engenty-wizards/shared/dist/marketplace-search.d.ts +20 -0
- package/node_modules/@engenty-wizards/shared/dist/marketplace-search.js +133 -0
- package/node_modules/@engenty-wizards/shared/dist/marketplace.d.ts +298 -0
- package/node_modules/@engenty-wizards/shared/dist/marketplace.js +211 -0
- package/node_modules/@engenty-wizards/shared/dist/merge.d.ts +7 -0
- package/node_modules/@engenty-wizards/shared/dist/merge.js +40 -0
- package/node_modules/@engenty-wizards/shared/dist/projects.d.ts +74 -0
- package/node_modules/@engenty-wizards/shared/dist/projects.js +96 -0
- package/node_modules/@engenty-wizards/shared/dist/run.d.ts +219 -0
- package/node_modules/@engenty-wizards/shared/dist/run.js +1 -0
- package/node_modules/@engenty-wizards/shared/dist/store.d.ts +100 -0
- package/node_modules/@engenty-wizards/shared/dist/store.js +44 -0
- package/node_modules/@engenty-wizards/shared/dist/workspace.d.ts +16 -0
- package/node_modules/@engenty-wizards/shared/dist/workspace.js +51 -0
- package/node_modules/@engenty-wizards/shared/package.json +10 -0
- package/package.json +62 -0
- package/plugin/.claude-plugin/plugin.json +9 -0
- package/plugin/.mcp.json +8 -0
- package/plugin/commands/wizard.md +8 -0
- package/plugin/skills/build-wizard/SKILL.md +27 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "7",
|
|
3
|
+
"dialect": "sqlite",
|
|
4
|
+
"entries": [
|
|
5
|
+
{
|
|
6
|
+
"idx": 0,
|
|
7
|
+
"version": "6",
|
|
8
|
+
"when": 1790945216913,
|
|
9
|
+
"tag": "0000_eminent_freak",
|
|
10
|
+
"breakpoints": true
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"idx": 1,
|
|
14
|
+
"version": "6",
|
|
15
|
+
"when": 1791067068249,
|
|
16
|
+
"tag": "0001_salty_master_chief",
|
|
17
|
+
"breakpoints": true
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
"idx": 2,
|
|
21
|
+
"version": "6",
|
|
22
|
+
"when": 1791071240061,
|
|
23
|
+
"tag": "0002_cloudy_firebird",
|
|
24
|
+
"breakpoints": true
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"idx": 3,
|
|
28
|
+
"version": "6",
|
|
29
|
+
"when": 1791096208798,
|
|
30
|
+
"tag": "0003_brown_warbound",
|
|
31
|
+
"breakpoints": true
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
"idx": 4,
|
|
35
|
+
"version": "6",
|
|
36
|
+
"when": 1791129806883,
|
|
37
|
+
"tag": "0004_romantic_nocturne",
|
|
38
|
+
"breakpoints": true
|
|
39
|
+
}
|
|
40
|
+
]
|
|
41
|
+
}
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
import { sql } from "drizzle-orm";
|
|
2
|
+
import { blob, index, integer, primaryKey, sqliteTable, text, uniqueIndex, } from "drizzle-orm/sqlite-core";
|
|
3
|
+
const now = sql `(unixepoch() * 1000)`;
|
|
4
|
+
const createdAt = () => integer("created_at", { mode: "timestamp_ms" }).notNull().default(now);
|
|
5
|
+
const updatedAt = () => integer("updated_at", { mode: "timestamp_ms" }).notNull().default(now);
|
|
6
|
+
export const project = sqliteTable("project", {
|
|
7
|
+
id: text("id").primaryKey(),
|
|
8
|
+
tenantId: text("tenant_id").notNull(),
|
|
9
|
+
name: text("name").notNull(),
|
|
10
|
+
brand: text("brand", { mode: "json" }).$type().notNull().default({}),
|
|
11
|
+
facts: text("facts", { mode: "json" }).$type().notNull().default([]),
|
|
12
|
+
mcpServers: text("mcp_servers", { mode: "json" })
|
|
13
|
+
.$type()
|
|
14
|
+
.notNull()
|
|
15
|
+
.default([]),
|
|
16
|
+
createdAt: createdAt(),
|
|
17
|
+
updatedAt: updatedAt(),
|
|
18
|
+
}, (t) => [index("project_tenant").on(t.tenantId)]);
|
|
19
|
+
export const wizard = sqliteTable("wizard", {
|
|
20
|
+
id: text("id").primaryKey(),
|
|
21
|
+
projectId: text("project_id")
|
|
22
|
+
.notNull()
|
|
23
|
+
.references(() => project.id, { onDelete: "cascade" }),
|
|
24
|
+
tenantId: text("tenant_id").notNull(),
|
|
25
|
+
title: text("title").notNull(),
|
|
26
|
+
draft: text("draft", { mode: "json" }).$type().notNull(),
|
|
27
|
+
publishedVersion: integer("published_version"),
|
|
28
|
+
shareToken: text("share_token").notNull(),
|
|
29
|
+
shareEnabled: integer("share_enabled", { mode: "boolean" }).notNull().default(true),
|
|
30
|
+
dailyRunLimit: integer("daily_run_limit").notNull().default(50),
|
|
31
|
+
starter: text("starter"),
|
|
32
|
+
/** The revision of the marketplace entry the wizard was made from. */
|
|
33
|
+
starterRevision: integer("starter_revision"),
|
|
34
|
+
/** Bumped on every draft write; a write carrying an older one is refused. */
|
|
35
|
+
revision: integer("revision").notNull().default(0),
|
|
36
|
+
createdAt: createdAt(),
|
|
37
|
+
updatedAt: updatedAt(),
|
|
38
|
+
}, (t) => [
|
|
39
|
+
uniqueIndex("wizard_share_token").on(t.shareToken),
|
|
40
|
+
index("wizard_project").on(t.projectId),
|
|
41
|
+
]);
|
|
42
|
+
export const wizardVersion = sqliteTable("wizard_version", {
|
|
43
|
+
id: text("id").primaryKey(),
|
|
44
|
+
wizardId: text("wizard_id")
|
|
45
|
+
.notNull()
|
|
46
|
+
.references(() => wizard.id, { onDelete: "cascade" }),
|
|
47
|
+
version: integer("version").notNull(),
|
|
48
|
+
definition: text("definition", { mode: "json" }).$type().notNull(),
|
|
49
|
+
/** The workspace as it was at publish time. */
|
|
50
|
+
files: text("files", { mode: "json" }).$type().notNull().default([]),
|
|
51
|
+
createdAt: createdAt(),
|
|
52
|
+
}, (t) => [uniqueIndex("wizard_version_n").on(t.wizardId, t.version)]);
|
|
53
|
+
/** The draft's workspace: widget code, libraries, reference data. Content lives in the blob store. */
|
|
54
|
+
export const wizardFile = sqliteTable("wizard_file", {
|
|
55
|
+
wizardId: text("wizard_id")
|
|
56
|
+
.notNull()
|
|
57
|
+
.references(() => wizard.id, { onDelete: "cascade" }),
|
|
58
|
+
path: text("path").notNull(),
|
|
59
|
+
hash: text("hash").notNull(),
|
|
60
|
+
mime: text("mime").notNull(),
|
|
61
|
+
size: integer("size").notNull(),
|
|
62
|
+
updatedAt: updatedAt(),
|
|
63
|
+
}, (t) => [primaryKey({ columns: [t.wizardId, t.path] })]);
|
|
64
|
+
export const wizardMessage = sqliteTable("wizard_message", {
|
|
65
|
+
id: text("id").primaryKey(),
|
|
66
|
+
wizardId: text("wizard_id")
|
|
67
|
+
.notNull()
|
|
68
|
+
.references(() => wizard.id, { onDelete: "cascade" }),
|
|
69
|
+
role: text("role", { enum: ["user", "assistant"] }).notNull(),
|
|
70
|
+
content: text("content").notNull(),
|
|
71
|
+
/** Set on assistant turns that changed the wizard. */
|
|
72
|
+
changed: integer("changed", { mode: "boolean" }).notNull().default(false),
|
|
73
|
+
/** "mcp": written by an admin's own client (Claude Code, Cursor …) named in `client`. */
|
|
74
|
+
source: text("source", { enum: ["studio", "mcp"] })
|
|
75
|
+
.notNull()
|
|
76
|
+
.default("studio"),
|
|
77
|
+
client: text("client"),
|
|
78
|
+
createdAt: createdAt(),
|
|
79
|
+
}, (t) => [index("wizard_message_wizard").on(t.wizardId, t.createdAt)]);
|
|
80
|
+
export const run = sqliteTable("run", {
|
|
81
|
+
id: text("id").primaryKey(),
|
|
82
|
+
wizardId: text("wizard_id")
|
|
83
|
+
.notNull()
|
|
84
|
+
.references(() => wizard.id, { onDelete: "cascade" }),
|
|
85
|
+
tenantId: text("tenant_id").notNull(),
|
|
86
|
+
/** The definition the run started on — edits never reach a running wizard. */
|
|
87
|
+
definition: text("definition", { mode: "json" }).$type().notNull(),
|
|
88
|
+
/** The workspace the run started with. */
|
|
89
|
+
files: text("files", { mode: "json" }).$type().notNull().default([]),
|
|
90
|
+
version: integer("version"),
|
|
91
|
+
mode: text("mode", { enum: ["test", "live"] }).notNull(),
|
|
92
|
+
visitorId: text("visitor_id"),
|
|
93
|
+
userId: text("user_id"),
|
|
94
|
+
ipHash: text("ip_hash"),
|
|
95
|
+
status: text("status", {
|
|
96
|
+
enum: ["waiting_input", "running", "done", "failed", "cancelled"],
|
|
97
|
+
}).notNull(),
|
|
98
|
+
cursor: text("cursor"),
|
|
99
|
+
state: text("state", { mode: "json" }).$type().notNull(),
|
|
100
|
+
error: text("error"),
|
|
101
|
+
/** What a running step is asking the person right now (a login, a code); null = nothing. */
|
|
102
|
+
ask: text("ask", { mode: "json" }).$type(),
|
|
103
|
+
costMicros: integer("cost_micros").notNull().default(0),
|
|
104
|
+
/** Set once the result is shared: `/s/<token>` shows it read-only. */
|
|
105
|
+
shareToken: text("share_token"),
|
|
106
|
+
sharedAt: integer("shared_at", { mode: "timestamp_ms" }),
|
|
107
|
+
/** Runs without an account are deleted after this; null keeps the run. */
|
|
108
|
+
expiresAt: integer("expires_at", { mode: "timestamp_ms" }),
|
|
109
|
+
createdAt: createdAt(),
|
|
110
|
+
updatedAt: updatedAt(),
|
|
111
|
+
}, (t) => [
|
|
112
|
+
uniqueIndex("run_share_token").on(t.shareToken),
|
|
113
|
+
index("run_expires").on(t.expiresAt),
|
|
114
|
+
index("run_wizard").on(t.wizardId, t.createdAt),
|
|
115
|
+
index("run_visitor").on(t.visitorId, t.createdAt),
|
|
116
|
+
index("run_status").on(t.status),
|
|
117
|
+
]);
|
|
118
|
+
export const runEvent = sqliteTable("run_event", {
|
|
119
|
+
id: integer("id").primaryKey({ autoIncrement: true }),
|
|
120
|
+
runId: text("run_id")
|
|
121
|
+
.notNull()
|
|
122
|
+
.references(() => run.id, { onDelete: "cascade" }),
|
|
123
|
+
stepId: text("step_id"),
|
|
124
|
+
type: text("type", { enum: ["step_started", "step_done", "tool", "info", "error"] }).notNull(),
|
|
125
|
+
message: text("message").notNull(),
|
|
126
|
+
/** A picture of the run the event is about; shown while the step works. */
|
|
127
|
+
assetId: text("asset_id"),
|
|
128
|
+
createdAt: createdAt(),
|
|
129
|
+
}, (t) => [index("run_event_run").on(t.runId, t.id)]);
|
|
130
|
+
/** What each step of a run cost, in credit micros: the measurements estimates are made from. */
|
|
131
|
+
export const runCost = sqliteTable("run_cost", {
|
|
132
|
+
runId: text("run_id")
|
|
133
|
+
.notNull()
|
|
134
|
+
.references(() => run.id, { onDelete: "cascade" }),
|
|
135
|
+
stepId: text("step_id").notNull(),
|
|
136
|
+
micros: integer("micros").notNull().default(0),
|
|
137
|
+
}, (t) => [primaryKey({ columns: [t.runId, t.stepId] })]);
|
|
138
|
+
export const asset = sqliteTable("asset", {
|
|
139
|
+
id: text("id").primaryKey(),
|
|
140
|
+
tenantId: text("tenant_id").notNull(),
|
|
141
|
+
runId: text("run_id"),
|
|
142
|
+
stepId: text("step_id"),
|
|
143
|
+
kind: text("kind").notNull(),
|
|
144
|
+
mime: text("mime").notNull(),
|
|
145
|
+
name: text("name").notNull(),
|
|
146
|
+
path: text("path").notNull(),
|
|
147
|
+
size: integer("size").notNull(),
|
|
148
|
+
createdAt: createdAt(),
|
|
149
|
+
}, (t) => [index("asset_run").on(t.runId)]);
|
|
150
|
+
/**
|
|
151
|
+
* A service imported as a connector, from the integrations registry or a pasted OpenAPI spec /
|
|
152
|
+
* MCP endpoint. The record is engenty's: actions, auth and source, normalised at import time.
|
|
153
|
+
*/
|
|
154
|
+
export const projectConnector = sqliteTable("project_connector", {
|
|
155
|
+
projectId: text("project_id")
|
|
156
|
+
.notNull()
|
|
157
|
+
.references(() => project.id, { onDelete: "cascade" }),
|
|
158
|
+
id: text("id").notNull(),
|
|
159
|
+
record: text("record", { mode: "json" }).$type().notNull(),
|
|
160
|
+
createdAt: createdAt(),
|
|
161
|
+
updatedAt: updatedAt(),
|
|
162
|
+
}, (t) => [primaryKey({ columns: [t.projectId, t.id] })]);
|
|
163
|
+
/**
|
|
164
|
+
* What a project holds for all its wizards: logos, assets (images, graphics, videos) and
|
|
165
|
+
* documents. `id` is the asset the content lives in. The first logo is the one end users see.
|
|
166
|
+
*/
|
|
167
|
+
export const projectFile = sqliteTable("project_file", {
|
|
168
|
+
id: text("id").primaryKey(),
|
|
169
|
+
projectId: text("project_id")
|
|
170
|
+
.notNull()
|
|
171
|
+
.references(() => project.id, { onDelete: "cascade" }),
|
|
172
|
+
kind: text("kind", { enum: ["logo", "asset", "document"] }).notNull(),
|
|
173
|
+
name: text("name").notNull(),
|
|
174
|
+
mime: text("mime").notNull(),
|
|
175
|
+
size: integer("size").notNull(),
|
|
176
|
+
description: text("description").notNull().default(""),
|
|
177
|
+
/** A web address the file was fetched from, when it was not uploaded. */
|
|
178
|
+
source: text("source"),
|
|
179
|
+
position: integer("position").notNull().default(0),
|
|
180
|
+
/** `pending` while it is read, described and indexed. */
|
|
181
|
+
status: text("status", { enum: ["pending", "ready", "failed"] })
|
|
182
|
+
.notNull()
|
|
183
|
+
.default("ready"),
|
|
184
|
+
error: text("error"),
|
|
185
|
+
/** A document's text as Markdown, in the blob store. */
|
|
186
|
+
textHash: text("text_hash"),
|
|
187
|
+
pages: integer("pages"),
|
|
188
|
+
chars: integer("chars"),
|
|
189
|
+
/** What the index holds of a document; `embeddings` includes the keywords. */
|
|
190
|
+
indexed: text("indexed", { enum: ["embeddings", "keywords"] }),
|
|
191
|
+
createdAt: createdAt(),
|
|
192
|
+
updatedAt: updatedAt(),
|
|
193
|
+
}, (t) => [index("project_file_project").on(t.projectId, t.kind, t.position)]);
|
|
194
|
+
/**
|
|
195
|
+
* The document index: passages of a project's documents. Keywords live in the FTS5 table
|
|
196
|
+
* `project_chunk_fts`, which triggers keep in step (see the migration); `embedding` is the
|
|
197
|
+
* passage's vector as float32 bytes, compared with libSQL's `vector_distance_cos`.
|
|
198
|
+
*/
|
|
199
|
+
export const projectChunk = sqliteTable("project_chunk", {
|
|
200
|
+
id: integer("id").primaryKey({ autoIncrement: true }),
|
|
201
|
+
projectId: text("project_id")
|
|
202
|
+
.notNull()
|
|
203
|
+
.references(() => project.id, { onDelete: "cascade" }),
|
|
204
|
+
fileId: text("file_id")
|
|
205
|
+
.notNull()
|
|
206
|
+
.references(() => projectFile.id, { onDelete: "cascade" }),
|
|
207
|
+
idx: integer("idx").notNull(),
|
|
208
|
+
text: text("text").notNull(),
|
|
209
|
+
embedding: blob("embedding", { mode: "buffer" }),
|
|
210
|
+
/** The model the vector was made with; only vectors of one model are compared. */
|
|
211
|
+
model: text("model"),
|
|
212
|
+
}, (t) => [
|
|
213
|
+
index("project_chunk_project").on(t.projectId, t.model),
|
|
214
|
+
index("project_chunk_file").on(t.fileId),
|
|
215
|
+
]);
|
|
216
|
+
// --- The wizard's store ------------------------------------------------------
|
|
217
|
+
// What a wizard keeps between runs for one person. `holder` is that person: "u:<userId>" for a
|
|
218
|
+
// signed-in admin, "v:<visitorId>" for an end user on a shared link.
|
|
219
|
+
/** One (wizard, person) pair that has stored something; `usedAt` drives the clean-up. */
|
|
220
|
+
export const storeHolder = sqliteTable("store_holder", {
|
|
221
|
+
wizardId: text("wizard_id")
|
|
222
|
+
.notNull()
|
|
223
|
+
.references(() => wizard.id, { onDelete: "cascade" }),
|
|
224
|
+
holder: text("holder").notNull(),
|
|
225
|
+
usedAt: integer("used_at", { mode: "timestamp_ms" }).notNull().default(now),
|
|
226
|
+
}, (t) => [primaryKey({ columns: [t.wizardId, t.holder] }), index("store_holder_used").on(t.usedAt)]);
|
|
227
|
+
/** A row of one of the wizard's lists. Columns are defined in the wizard, values live here. */
|
|
228
|
+
export const storeRow = sqliteTable("store_row", {
|
|
229
|
+
id: text("id").primaryKey(),
|
|
230
|
+
wizardId: text("wizard_id")
|
|
231
|
+
.notNull()
|
|
232
|
+
.references(() => wizard.id, { onDelete: "cascade" }),
|
|
233
|
+
holder: text("holder").notNull(),
|
|
234
|
+
list: text("list").notNull(),
|
|
235
|
+
/** The key column's value, normalised; saving a row with a known key updates that row. */
|
|
236
|
+
key: text("key"),
|
|
237
|
+
cells: text("cells", { mode: "json" }).$type().notNull(),
|
|
238
|
+
createdAt: createdAt(),
|
|
239
|
+
updatedAt: updatedAt(),
|
|
240
|
+
}, (t) => [
|
|
241
|
+
uniqueIndex("store_row_key").on(t.wizardId, t.holder, t.list, t.key),
|
|
242
|
+
index("store_row_list").on(t.wizardId, t.holder, t.list, t.createdAt),
|
|
243
|
+
]);
|
|
244
|
+
/** A file the wizard keeps (a downloaded invoice, a scan). Content lives in the blob store. */
|
|
245
|
+
export const storeFile = sqliteTable("store_file", {
|
|
246
|
+
wizardId: text("wizard_id")
|
|
247
|
+
.notNull()
|
|
248
|
+
.references(() => wizard.id, { onDelete: "cascade" }),
|
|
249
|
+
holder: text("holder").notNull(),
|
|
250
|
+
path: text("path").notNull(),
|
|
251
|
+
hash: text("hash").notNull(),
|
|
252
|
+
mime: text("mime").notNull(),
|
|
253
|
+
size: integer("size").notNull(),
|
|
254
|
+
source: text("source"),
|
|
255
|
+
updatedAt: updatedAt(),
|
|
256
|
+
}, (t) => [primaryKey({ columns: [t.wizardId, t.holder, t.path] })]);
|
|
257
|
+
/** A connected account or a kept browser session. `data` is encrypted. */
|
|
258
|
+
export const storeSecret = sqliteTable("store_secret", {
|
|
259
|
+
id: text("id").primaryKey(),
|
|
260
|
+
wizardId: text("wizard_id")
|
|
261
|
+
.notNull()
|
|
262
|
+
.references(() => wizard.id, { onDelete: "cascade" }),
|
|
263
|
+
holder: text("holder").notNull(),
|
|
264
|
+
/** "connection:<id>" or "browser:<host>". */
|
|
265
|
+
slot: text("slot").notNull(),
|
|
266
|
+
provider: text("provider").notNull(),
|
|
267
|
+
/** Shown to the person: the account's address or the site's name. */
|
|
268
|
+
label: text("label").notNull(),
|
|
269
|
+
data: text("data").notNull(),
|
|
270
|
+
createdAt: createdAt(),
|
|
271
|
+
updatedAt: updatedAt(),
|
|
272
|
+
}, (t) => [uniqueIndex("store_secret_slot").on(t.wizardId, t.holder, t.slot)]);
|
|
273
|
+
/** Marketplace entries starred here: the runtime keeps them for offline use. */
|
|
274
|
+
export const marketplaceStar = sqliteTable("marketplace_star", {
|
|
275
|
+
tenantId: text("tenant_id").notNull(),
|
|
276
|
+
entryId: text("entry_id").notNull(),
|
|
277
|
+
createdAt: createdAt(),
|
|
278
|
+
}, (t) => [primaryKey({ columns: [t.tenantId, t.entryId] })]);
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { existsSync, mkdirSync } from "node:fs";
|
|
3
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { generateText, Output } from "ai";
|
|
6
|
+
import PostalMime from "postal-mime";
|
|
7
|
+
import { z } from "zod";
|
|
8
|
+
import { parseCsvMatrix } from "../engenty/csv-import/csv-matrix.js";
|
|
9
|
+
import { readXlsxWorkbook } from "../engenty/csv-import/xlsx-workbook.js";
|
|
10
|
+
import { splitMarkdownByPageBreaks } from "../engenty/doc-converter/page-break.js";
|
|
11
|
+
import { LocalProvider } from "../engenty/doc-converter/providers/local/index.js";
|
|
12
|
+
import { invoiceSchema, receiptSchema } from "../engenty/document-scanner/schemas/index.js";
|
|
13
|
+
import { env } from "../env.js";
|
|
14
|
+
import { costOf, textModel } from "../models.js";
|
|
15
|
+
import { htmlToMarkdown } from "../render/convert.js";
|
|
16
|
+
export class UnreadableDocument extends Error {
|
|
17
|
+
}
|
|
18
|
+
const DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
|
|
19
|
+
const XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
|
|
20
|
+
const MIME_BY_EXT = {
|
|
21
|
+
pdf: "application/pdf",
|
|
22
|
+
docx: DOCX,
|
|
23
|
+
doc: "application/msword",
|
|
24
|
+
xlsx: XLSX,
|
|
25
|
+
csv: "text/csv",
|
|
26
|
+
tsv: "text/csv",
|
|
27
|
+
txt: "text/plain",
|
|
28
|
+
md: "text/markdown",
|
|
29
|
+
json: "application/json",
|
|
30
|
+
html: "text/html",
|
|
31
|
+
htm: "text/html",
|
|
32
|
+
eml: "message/rfc822",
|
|
33
|
+
png: "image/png",
|
|
34
|
+
jpg: "image/jpeg",
|
|
35
|
+
jpeg: "image/jpeg",
|
|
36
|
+
webp: "image/webp",
|
|
37
|
+
gif: "image/gif",
|
|
38
|
+
};
|
|
39
|
+
/** The type to read a file as: mail servers and browsers often only say "octet-stream". */
|
|
40
|
+
export function documentMime(name, mime) {
|
|
41
|
+
const declared = mime.split(";")[0].trim().toLowerCase();
|
|
42
|
+
if (declared && declared !== "application/octet-stream") {
|
|
43
|
+
return declared;
|
|
44
|
+
}
|
|
45
|
+
return MIME_BY_EXT[name.split(".").pop()?.toLowerCase() ?? ""] ?? declared;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Reads scans and photos with a vision model, as an engenty doc-converter provider. Pages are
|
|
49
|
+
* marked with the converter's page-break sentinel so paged output looks like any other.
|
|
50
|
+
*/
|
|
51
|
+
class VisionProvider {
|
|
52
|
+
charge;
|
|
53
|
+
signal;
|
|
54
|
+
call;
|
|
55
|
+
id = "vision";
|
|
56
|
+
name = "Vision model";
|
|
57
|
+
supportedTypes = [
|
|
58
|
+
"application/pdf",
|
|
59
|
+
"image/png",
|
|
60
|
+
"image/jpeg",
|
|
61
|
+
"image/webp",
|
|
62
|
+
"image/gif",
|
|
63
|
+
];
|
|
64
|
+
constructor(charge, signal, call) {
|
|
65
|
+
this.charge = charge;
|
|
66
|
+
this.signal = signal;
|
|
67
|
+
this.call = call;
|
|
68
|
+
}
|
|
69
|
+
canConvert(mimeType) {
|
|
70
|
+
return this.supportedTypes.includes(mimeType);
|
|
71
|
+
}
|
|
72
|
+
async convert(data, filename, mimeType, _options) {
|
|
73
|
+
// Reads scans and photos: the class it runs on must take PDFs and images.
|
|
74
|
+
const reader = await textModel("standard", this.call);
|
|
75
|
+
const result = await generateText({
|
|
76
|
+
model: reader.model,
|
|
77
|
+
abortSignal: this.signal,
|
|
78
|
+
maxOutputTokens: 16_000,
|
|
79
|
+
system: [
|
|
80
|
+
"You transcribe documents into clean Markdown, faithfully and completely.",
|
|
81
|
+
"Keep every number, date, name and amount exactly as printed. Tables become Markdown tables.",
|
|
82
|
+
"Do not summarise, do not comment, do not wrap the result in a code fence.",
|
|
83
|
+
'For a PDF with several pages, start each page with this line: <page-break number="N" total="T"></page-break>',
|
|
84
|
+
"If a part is illegible, write [unleserlich] there.",
|
|
85
|
+
"A photo that is no document (a room, an object, a damage): describe precisely what it shows instead — what is there, how many, materials, condition, visible defects — without guessing what cannot be seen.",
|
|
86
|
+
].join("\n"),
|
|
87
|
+
messages: [
|
|
88
|
+
{
|
|
89
|
+
role: "user",
|
|
90
|
+
content: [
|
|
91
|
+
{ type: "file", data, mediaType: mimeType, filename: filename || "document" },
|
|
92
|
+
{ type: "text", text: "Transcribe this document." },
|
|
93
|
+
],
|
|
94
|
+
},
|
|
95
|
+
],
|
|
96
|
+
});
|
|
97
|
+
await this.charge?.(costOf(reader, result.usage));
|
|
98
|
+
const markdown = result.text.trim();
|
|
99
|
+
return {
|
|
100
|
+
markdown,
|
|
101
|
+
metadata: { word_count: markdown.split(/\s+/).filter(Boolean).length },
|
|
102
|
+
source: { filename, mime_type: mimeType, size_bytes: data.byteLength },
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
const local = new LocalProvider();
|
|
107
|
+
function tableMarkdown(columns, rows, limit = 400) {
|
|
108
|
+
const cell = (v) => v.replace(/\|/g, "\\|").replace(/\s+/g, " ").trim();
|
|
109
|
+
const head = `| ${columns.map(cell).join(" | ")} |\n|${columns.map(() => " --- ").join("|")}|`;
|
|
110
|
+
const body = rows.slice(0, limit).map((r) => `| ${r.map(cell).join(" | ")} |`);
|
|
111
|
+
const more = rows.length > limit ? [`\n… ${rows.length - limit} more rows`] : [];
|
|
112
|
+
return [head, ...body, ...more].join("\n");
|
|
113
|
+
}
|
|
114
|
+
async function parseMail(data) {
|
|
115
|
+
const mail = await PostalMime.parse(data);
|
|
116
|
+
const body = mail.text?.trim() || (mail.html ? htmlToMarkdown(mail.html) : "");
|
|
117
|
+
return [
|
|
118
|
+
`From: ${mail.from?.address ?? ""}`,
|
|
119
|
+
`To: ${(mail.to ?? []).map((t) => t.address).join(", ")}`,
|
|
120
|
+
`Date: ${mail.date ?? ""}`,
|
|
121
|
+
`Subject: ${mail.subject ?? ""}`,
|
|
122
|
+
mail.attachments.length
|
|
123
|
+
? `Attachments: ${mail.attachments.map((a) => a.filename ?? "unnamed").join(", ")}`
|
|
124
|
+
: "",
|
|
125
|
+
"",
|
|
126
|
+
body,
|
|
127
|
+
]
|
|
128
|
+
.filter((line, i) => line || i === 5)
|
|
129
|
+
.join("\n");
|
|
130
|
+
}
|
|
131
|
+
const CACHE_DIR = join(env.dataDir, "parsed");
|
|
132
|
+
mkdirSync(CACHE_DIR, { recursive: true });
|
|
133
|
+
/**
|
|
134
|
+
* Turns a file into Markdown: PDFs and Word files by their own text, scans and photos by a
|
|
135
|
+
* vision model, spreadsheets and CSV as tables, saved mails with their headers. The result is
|
|
136
|
+
* kept by content, so a file is read (and paid for) once.
|
|
137
|
+
*/
|
|
138
|
+
export async function parseDocument(input, opts = {}) {
|
|
139
|
+
const mime = documentMime(input.name, input.mime);
|
|
140
|
+
return kept(input.data, mime, () => read({ ...input, mime }, opts));
|
|
141
|
+
}
|
|
142
|
+
/** What was made of a file, kept by its content: the same bytes are never worked through twice. */
|
|
143
|
+
async function kept(data, variant, make) {
|
|
144
|
+
const file = join(CACHE_DIR, `${createHash("sha256").update(data).update(variant).digest("hex")}.json`);
|
|
145
|
+
if (existsSync(file)) {
|
|
146
|
+
return JSON.parse(await readFile(file, "utf8"));
|
|
147
|
+
}
|
|
148
|
+
const made = await make();
|
|
149
|
+
await writeFile(file, JSON.stringify(made));
|
|
150
|
+
return made;
|
|
151
|
+
}
|
|
152
|
+
async function read(input, opts) {
|
|
153
|
+
const { data, name, mime } = input;
|
|
154
|
+
const vision = new VisionProvider(opts.charge, opts.signal, opts.call);
|
|
155
|
+
if (mime.startsWith("image/")) {
|
|
156
|
+
if (!vision.canConvert(mime)) {
|
|
157
|
+
throw new UnreadableDocument(`Bilder vom Typ ${mime} kann ich nicht lesen (PNG, JPEG, WebP gehen).`);
|
|
158
|
+
}
|
|
159
|
+
const result = await vision.convert(data, name, mime);
|
|
160
|
+
return { markdown: result.markdown, pages: 1, via: "vision" };
|
|
161
|
+
}
|
|
162
|
+
if (mime === "application/pdf") {
|
|
163
|
+
const result = await local.convert(data, name, mime);
|
|
164
|
+
const pages = result.metadata.page_count ?? null;
|
|
165
|
+
const letters = result.markdown.replace(/<page-break[^>]*><\/page-break>|\s/g, "").length;
|
|
166
|
+
// Hardly any text per page: the PDF is a scan.
|
|
167
|
+
if (letters < 60 * Math.max(1, pages ?? 1)) {
|
|
168
|
+
const seen = await vision.convert(data, name, mime);
|
|
169
|
+
return {
|
|
170
|
+
markdown: seen.markdown,
|
|
171
|
+
pages: pages ?? (splitMarkdownByPageBreaks(seen.markdown).length || null),
|
|
172
|
+
via: "vision",
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
return { markdown: result.markdown, pages, via: "text" };
|
|
176
|
+
}
|
|
177
|
+
if (local.canConvert(mime)) {
|
|
178
|
+
return { markdown: (await local.convert(data, name, mime)).markdown, pages: null, via: "text" };
|
|
179
|
+
}
|
|
180
|
+
if (mime === XLSX) {
|
|
181
|
+
const workbook = await readXlsxWorkbook(data);
|
|
182
|
+
const sheets = workbook.sheets
|
|
183
|
+
.filter((s) => s.matrix.rows.length)
|
|
184
|
+
.map((s) => `## ${s.name}\n\n${tableMarkdown(s.matrix.columns, s.matrix.rows)}`);
|
|
185
|
+
return { markdown: sheets.join("\n\n"), pages: null, via: "table" };
|
|
186
|
+
}
|
|
187
|
+
const text = () => Buffer.from(data).toString("utf8");
|
|
188
|
+
if (mime === "text/csv") {
|
|
189
|
+
const matrix = parseCsvMatrix(text());
|
|
190
|
+
return { markdown: tableMarkdown(matrix.columns, matrix.rows), pages: null, via: "table" };
|
|
191
|
+
}
|
|
192
|
+
if (mime === "text/html") {
|
|
193
|
+
return { markdown: htmlToMarkdown(text()), pages: null, via: "text" };
|
|
194
|
+
}
|
|
195
|
+
if (mime === "message/rfc822") {
|
|
196
|
+
return { markdown: await parseMail(data), pages: null, via: "mail" };
|
|
197
|
+
}
|
|
198
|
+
if (mime === "application/json" || mime.startsWith("text/")) {
|
|
199
|
+
return { markdown: text(), pages: null, via: "text" };
|
|
200
|
+
}
|
|
201
|
+
throw new UnreadableDocument(`„${name}“ (${mime}) kann ich nicht lesen.`);
|
|
202
|
+
}
|
|
203
|
+
// --- structured extraction -----------------------------------------------------
|
|
204
|
+
/** The scanner's schemas say nothing about the currency; SaaS invoices come in several. */
|
|
205
|
+
const currency = z
|
|
206
|
+
.string()
|
|
207
|
+
.nullable()
|
|
208
|
+
.describe("ISO 4217 code of the currency the amounts are printed in (EUR, USD, …)");
|
|
209
|
+
const SCHEMAS = {
|
|
210
|
+
invoice: invoiceSchema.extend({ currency }),
|
|
211
|
+
receipt: receiptSchema.extend({ currency }),
|
|
212
|
+
};
|
|
213
|
+
/**
|
|
214
|
+
* Pulls the fields of an invoice or receipt out of a document, with engenty's document-scanner
|
|
215
|
+
* schemas (their field descriptions are the extraction instructions).
|
|
216
|
+
*/
|
|
217
|
+
export async function scanDocument(input, kind, opts = {}) {
|
|
218
|
+
const doc = await parseDocument(input, opts);
|
|
219
|
+
return kept(input.data, `scan:${kind}`, async () => {
|
|
220
|
+
const extractor = await textModel("classifier", opts.call);
|
|
221
|
+
const result = await generateText({
|
|
222
|
+
model: extractor.model,
|
|
223
|
+
abortSignal: opts.signal,
|
|
224
|
+
output: Output.object({ schema: SCHEMAS[kind] }),
|
|
225
|
+
prompt: `Extract the ${kind} from this document. Amounts are numbers in the document's own currency, exactly as printed; never convert or compute them.\n\nFile: ${input.name}\n\n${doc.markdown.slice(0, 30_000)}`,
|
|
226
|
+
});
|
|
227
|
+
await opts.charge?.(costOf(extractor, result.usage));
|
|
228
|
+
return result.output;
|
|
229
|
+
});
|
|
230
|
+
}
|