pict-section-dataimport 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (17) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +108 -0
  3. package/package.json +80 -0
  4. package/source/Pict-Section-DataImport-CSS.js +73 -0
  5. package/source/Pict-Section-DataImport.js +54 -0
  6. package/source/providers/Pict-Provider-DataImport-Meadow.js +192 -0
  7. package/source/providers/Pict-Provider-DataImport.js +149 -0
  8. package/source/seams/DataImport-Parser-Delimited.js +128 -0
  9. package/source/seams/DataImport-Parser-FixedWidth.js +87 -0
  10. package/source/seams/DataImport-Parser-Xlsx.js +81 -0
  11. package/source/seams/DataImport-ParserProvider.js +123 -0
  12. package/source/seams/DataImport-PushTarget.js +82 -0
  13. package/source/seams/DataImport-SchemaProvider.js +72 -0
  14. package/source/seams/DataImport-StateStore.js +123 -0
  15. package/source/services/DataImport-ComprehensionBuilder.js +123 -0
  16. package/source/services/DataImport-Session.js +101 -0
  17. package/source/views/PictView-DataImport-Wizard.js +657 -0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Steven Velozo
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,108 @@
1
+ # pict-section-dataimport
2
+
3
+ An embeddable, configurable **data-import wizard** for Pict apps. A host registers the provider and
4
+ calls `createImportWizard(hash, config)` to drop the whole flow into its UI:
5
+
6
+ 1. **Upload** a CSV / TSV / Excel / fixed-width file ([pict-section-upload](../pict-section-upload)).
7
+ 2. **Validate** — detect columns, sample rows, fixed-width boundaries.
8
+ 3. **Map** source columns to a target schema — a live **Meadow** entity schema, or a **complex
9
+ non-Meadow schema supplied in host config**.
10
+ 4. **Generate** comprehensions — reusing [meadow-integration](../../meadow/meadow-integration)'s
11
+ browser-safe transform engine (Solvers + one-row-to-many fan-out).
12
+ 5. **Push** — records via the browser `EntityProvider` (GUID marshaling + FK resolution), **or** one
13
+ POST of the whole comprehension to a `/Comprehension/Push` endpoint.
14
+
15
+ The step container is [pict-section-accordion](../pict-section-accordion) (wizard / accordion /
16
+ stepper). Everything follows the [Pict theme criteria](../pict-section-theme) — themed CSS at
17
+ priority 500.
18
+
19
+ ## Install
20
+
21
+ ```bash
22
+ npm install pict-section-dataimport
23
+ ```
24
+
25
+ Peers: `pict`, `pict-section-upload`, `pict-section-accordion`, and `meadow-integration` (its
26
+ browser-safe engine re-export is used to build comprehensions — see "Engine" below). Optional peers:
27
+ `xlsx` (only for Excel import), `pict-section-modal`, `pict-section-form`, `pict-section-recordset`.
28
+
29
+ ## Usage
30
+
31
+ ```javascript
32
+ const libDataImport = require('pict-section-dataimport');
33
+ pict.addProvider('Pict-Section-DataImport', libDataImport.default_configuration, libDataImport);
34
+
35
+ pict.providers['Pict-Section-DataImport'].createImportWizard('BooksImport',
36
+ {
37
+ DestinationAddress: '#BooksImport',
38
+ RenderMode: 'wizard', // 'wizard' | 'accordion' | 'stepper'
39
+
40
+ // Target schema (step 3)
41
+ SchemaSource: 'meadow', // 'meadow' | 'config' | a SchemaProvider | (ctx)=>Promise
42
+ MeadowEntities: [ 'Book', 'Author', 'BookAuthorJoin' ], // when 'meadow'
43
+ Schema: { Entities: [ ... ], Order: [ ... ] }, // when 'config'
44
+ URLPrefix: '/1.0/',
45
+
46
+ // Push (step 5)
47
+ PushMode: 'entityprovider', // 'entityprovider' | 'comprehension' | a PushTarget | (comp,ctx)=>Promise
48
+ GUIDPrefix: 'BOOKSIMPORT', // pin per app/env — same prefix => idempotent upserts
49
+ AllowGUIDTruncation: true, // truncate the prefix if a marshaled GUID exceeds the column
50
+ ComprehensionPushURL: '/1.0/Comprehension/Push', // when PushMode 'comprehension'
51
+
52
+ // A starting mapping the user can edit (entities, GUID templates, fan-out Solvers)
53
+ DefaultMapping: { Order: [...], Entities: { ... } },
54
+
55
+ // Persistence (resume a partially-mapped session)
56
+ StateStore: 'memory', // 'memory' | 'localStorage' | a StateStore
57
+ PersistenceHook: { save, load, list, remove }, // overrides StateStore (server persistence)
58
+
59
+ AllowedFileTypes: [ 'csv', 'tsv', 'xlsx', 'fixedwidth' ],
60
+ OnStepChange, OnSchemaLoaded, OnComprehension, OnPushComplete, OnError,
61
+ });
62
+ pict.views['BooksImport'].render();
63
+ ```
64
+
65
+ ## Seams (all host-swappable)
66
+
67
+ | Seam | Built-ins | Swap by |
68
+ |---|---|---|
69
+ | **ParserProvider** | csv, tsv, fixed-width, xlsx (lazy) | `Parsers: { kind: instance }` |
70
+ | **SchemaProvider** | Meadow `/Schema` fetch, host-config descriptor | `SchemaSource: instance \| (ctx)=>Promise` |
71
+ | **PushTarget** | EntityProvider records, Comprehension POST | `PushMode: instance \| (comp,ctx)=>Promise` |
72
+ | **StateStore** | memory, localStorage, PersistenceHook | `StateStore: instance` |
73
+
74
+ The mapping → comprehension step + the EntityProvider push reuse `meadow-integration`'s engine; both
75
+ are reached through the browser-clean `Meadow-Integration-Engine.js` re-export, so **no server-only
76
+ code (orator / the meadow ORM / DB drivers / xlsx) enters the bundle** — there's a test that asserts this.
77
+
78
+ ## Comprehension generation (the core)
79
+
80
+ `DataImportComprehensionBuilder.build(mapping, rows)` runs the canonical
81
+ `MeadowIntegrationTabularTransform.transformRecord` per row, so the output is byte-identical to what
82
+ the `meadow-integration` CLI produces from the same mapping files — including `Solvers` +
83
+ `MultipleGUIDUniqueness` fan-out (one CSV row → a Book + N Authors + N joins). Entities are emitted in
84
+ `Mapping.Order` (referenced-before-referrer); `validateForeignKeyOrder()` flags any FK that would
85
+ resolve to NULL.
86
+
87
+ ## Examples
88
+
89
+ - **`example_applications/books_import`** — imports a books CSV into the quackage sqlite bookstore
90
+ harness (Meadow schema, EntityProvider push, the bookstore fan-out mapping). Verified end-to-end:
91
+ 3 rows → 3 Books + 4 Authors + 5 joins with resolved foreign keys, idempotent on re-run.
92
+ - **`example_applications/archive_import`** — generates **Archive.org-shaped** comprehensions for a
93
+ complex, non-Meadow schema defined entirely in app config: `ArchiveItem` ↔ `ArchiveCollection`
94
+ many-to-many (fan-out from a `;`-delimited collection column), modeled on a real archive.org item
95
+ (the public-domain General Mills Monster Cereals commercials). Generate, preview, download the JSON.
96
+
97
+ ## Test
98
+
99
+ ```bash
100
+ npm test # mocha TDD — core engine integration, parsers, seams, dependency boundary, wizard wiring
101
+ ```
102
+
103
+ ## Note on the mapping / preview UI
104
+
105
+ v1 ships the wizard's own themed, pict-templated mapping editor (per-field source-column selects +
106
+ a raw-JSON escape hatch for fan-out/FK rules) and a generated-record preview table. Swapping in
107
+ `pict-section-form` (descriptor-driven mapping) or `pict-section-recordset` (paged preview) is a
108
+ documented next step — the seams + serializable session state make it additive.
package/package.json ADDED
@@ -0,0 +1,80 @@
1
+ {
2
+ "name": "pict-section-dataimport",
3
+ "version": "1.0.0",
4
+ "description": "Embeddable, configurable data-import wizard for Pict apps — upload a CSV/TSV/Excel/fixed-width file, validate + detect its schema, map columns to a Meadow (or host-config) entity schema, generate comprehensions, and push them (records via the browser EntityProvider, or one POST to a Comprehension endpoint). Composes pict-section-upload + pict-section-accordion + pict-section-form, reusing meadow-integration's browser-safe transform/push engine.",
5
+ "main": "source/Pict-Section-DataImport.js",
6
+ "files": [
7
+ "source"
8
+ ],
9
+ "scripts": {
10
+ "test": "npx mocha -u tdd -R spec",
11
+ "tests": "npx mocha -u tdd -R spec -g",
12
+ "start": "node source/Pict-Section-DataImport.js",
13
+ "coverage": "./node_modules/.bin/nyc --reporter=lcov --reporter=text-lcov ./node_modules/mocha/bin/_mocha -- -u tdd -R spec",
14
+ "build": "npx quack build"
15
+ },
16
+ "repository": {
17
+ "type": "git",
18
+ "url": "git+https://github.com/fable-retold/pict-section-dataimport.git"
19
+ },
20
+ "author": "steven velozo <steven@velozo.com>",
21
+ "license": "MIT",
22
+ "bugs": {
23
+ "url": "https://github.com/fable-retold/pict-section-dataimport/issues"
24
+ },
25
+ "homepage": "https://github.com/fable-retold/pict-section-dataimport#readme",
26
+ "mocha": {
27
+ "diff": true,
28
+ "extension": [
29
+ "js"
30
+ ],
31
+ "package": "./package.json",
32
+ "reporter": "spec",
33
+ "slow": "200",
34
+ "timeout": "10000",
35
+ "ui": "tdd",
36
+ "watch-files": [
37
+ "source/**/*.js",
38
+ "test/**/*.js"
39
+ ]
40
+ },
41
+ "dependencies": {
42
+ "fable-serviceproviderbase": "^3.0.19",
43
+ "pict-provider": "^1.0.13",
44
+ "pict-view": "^1.0.68"
45
+ },
46
+ "peerDependencies": {
47
+ "meadow-integration": ">=1.0.43",
48
+ "pict": ">=1.0.300",
49
+ "pict-section-accordion": ">=1.0.0",
50
+ "pict-section-form": ">=1.0.0",
51
+ "pict-section-modal": ">=1.0.0",
52
+ "pict-section-recordset": ">=1.0.0",
53
+ "pict-section-upload": ">=1.0.0",
54
+ "xlsx": ">=0.18.0"
55
+ },
56
+ "peerDependenciesMeta": {
57
+ "pict-section-form": {
58
+ "optional": true
59
+ },
60
+ "pict-section-recordset": {
61
+ "optional": true
62
+ },
63
+ "pict-section-modal": {
64
+ "optional": true
65
+ },
66
+ "xlsx": {
67
+ "optional": true
68
+ }
69
+ },
70
+ "devDependencies": {
71
+ "@types/mocha": "^10.0.10",
72
+ "@types/node": "^16.18.126",
73
+ "browser-env": "^3.3.0",
74
+ "chai": "^4.3.10",
75
+ "mocha": "^10.4.0",
76
+ "pict": "^1.0.373",
77
+ "pict-application": "^1.0.34",
78
+ "quackage": "^1.3.0"
79
+ }
80
+ }
package/source/Pict-Section-DataImport-CSS.js ADDED
@@ -0,0 +1,73 @@
1
+ // Themeable wizard CSS for pict-section-dataimport. Registered once (by hash) by the provider. Covers
2
+ // the per-step body content (detect/columns/mapping/preview/push); the wizard chrome itself comes from
3
+ // pict-section-accordion, and the dropzone from pict-section-upload. Brand via the --theme-* tokens.
4
+
5
+ module.exports = /*css*/`
6
+ .psd { width: 100%; box-sizing: border-box; font-size: var(--theme-typography-size-md, 0.95rem); color: var(--theme-color-text-primary, #1f2733); }
7
+ .psd *, .psd *::before, .psd *::after { box-sizing: border-box; }
8
+ .psd-step { display: flex; flex-direction: column; gap: var(--theme-spacing-md, 0.85rem); }
9
+ .psd-hint { font-size: var(--theme-typography-size-sm, 0.86rem); color: var(--theme-color-text-muted, #6b7686); margin: 0; }
10
+ .psd-section-title { font-size: var(--theme-typography-size-md, 0.95rem); font-weight: var(--theme-typography-weight-bold, 700); margin: 0; }
11
+
12
+ /* Parse options row (delimiter, has-header, kind) */
13
+ .psd-options { display: flex; flex-wrap: wrap; align-items: center; gap: var(--theme-spacing-md, 0.9rem); }
14
+ .psd-field { display: flex; align-items: center; gap: 0.4rem; font-size: var(--theme-typography-size-sm, 0.86rem); }
15
+ .psd-field label { color: var(--theme-color-text-secondary, #45596b); }
16
+ .psd-input, .psd-select { font: inherit; font-size: var(--theme-typography-size-sm, 0.88rem); padding: 0.35rem 0.5rem;
17
+ border: 1px solid var(--theme-color-border-default, #d7dce3); border-radius: var(--theme-radius-md, 6px);
18
+ background: var(--theme-color-background-primary, #fff); color: var(--theme-color-text-primary, #1f2733); }
19
+ .psd-input:focus, .psd-select:focus { outline: none; border-color: var(--theme-color-brand-primary, #156dd1);
20
+ box-shadow: 0 0 0 3px color-mix(in srgb, var(--theme-color-focus-outline, #156dd1) 22%, transparent); }
21
+ .psd-textarea { width: 100%; min-height: 150px; font-family: var(--theme-typography-family-mono, ui-monospace, monospace);
22
+ font-size: var(--theme-typography-size-sm, 0.82rem); padding: 0.6rem 0.7rem; border-radius: var(--theme-radius-md, 6px);
23
+ border: 1px solid var(--theme-color-border-default, #d7dce3); background: var(--theme-color-background-secondary, #f6f7f9);
24
+ color: var(--theme-color-text-primary, #1f2733); resize: vertical; }
25
+
26
+ /* Tables (detected columns, sample rows, generated-record preview) */
27
+ .psd-table-wrap { overflow: auto; border: 1px solid var(--theme-color-border-light, #e8ebf0); border-radius: var(--theme-radius-md, 6px); max-height: 320px; }
28
+ .psd-table { border-collapse: collapse; width: 100%; font-size: var(--theme-typography-size-sm, 0.84rem); }
29
+ .psd-table th { position: sticky; top: 0; text-align: left; padding: 0.45rem 0.6rem; white-space: nowrap;
30
+ background: var(--theme-color-background-secondary, #f6f7f9); color: var(--theme-color-text-secondary, #45596b);
31
+ border-bottom: 1px solid var(--theme-color-border-default, #d7dce3); font-weight: var(--theme-typography-weight-medium, 600); }
32
+ .psd-table td { padding: 0.4rem 0.6rem; border-bottom: 1px solid var(--theme-color-border-light, #eef1f5);
33
+ color: var(--theme-color-text-primary, #1f2733); white-space: nowrap; max-width: 320px; overflow: hidden; text-overflow: ellipsis; }
34
+ .psd-table tr:last-child td { border-bottom: none; }
35
+ .psd-type { font-size: var(--theme-typography-size-xs, 0.72rem); color: var(--theme-color-text-muted, #6b7686); font-weight: 400; }
36
+ .psd-mono { font-family: var(--theme-typography-family-mono, ui-monospace, monospace); }
37
+ .psd-ruler { font-family: var(--theme-typography-family-mono, ui-monospace, monospace); font-size: var(--theme-typography-size-sm, 0.82rem);
38
+ white-space: pre; overflow-x: auto; padding: 0.5rem 0.6rem; background: var(--theme-color-background-secondary, #f6f7f9);
39
+ border: 1px solid var(--theme-color-border-light, #e8ebf0); border-radius: var(--theme-radius-md, 6px); }
40
+
41
+ /* Mapping editor */
42
+ .psd-entity { border: 1px solid var(--theme-color-border-light, #e8ebf0); border-radius: var(--theme-radius-md, 8px); padding: var(--theme-spacing-md, 0.85rem); }
43
+ .psd-entity-head { display: flex; align-items: center; justify-content: space-between; gap: 0.6rem; margin-bottom: 0.6rem; }
44
+ .psd-entity-name { font-weight: var(--theme-typography-weight-bold, 700); }
45
+ .psd-entity-count { font-size: var(--theme-typography-size-xs, 0.74rem); color: var(--theme-color-text-muted, #6b7686); }
46
+ .psd-map-row { display: grid; grid-template-columns: 1fr 1fr; gap: 0.5rem; align-items: center; padding: 0.25rem 0; }
47
+ .psd-map-field { font-size: var(--theme-typography-size-sm, 0.86rem); }
48
+ .psd-map-required { color: var(--theme-color-status-error, #b62828); }
49
+
50
+ /* Buttons + status */
51
+ .psd-btn { cursor: pointer; font: inherit; font-size: var(--theme-typography-size-sm, 0.88rem); font-weight: var(--theme-typography-weight-medium, 600);
52
+ padding: 0.45rem 0.9rem; border-radius: var(--theme-radius-md, 6px); border: none;
53
+ background: var(--theme-color-brand-primary, #156dd1); color: var(--theme-color-background-panel, #fff); display: inline-flex; align-items: center; gap: 0.4rem; }
54
+ .psd-btn:hover { background: var(--theme-color-brand-primaryhover, #1257ab); }
55
+ .psd-btn-ghost { background: transparent; color: var(--theme-color-brand-primary, #156dd1); border: 1px solid var(--theme-color-border-default, #d7dce3); }
56
+ .psd-btn-ghost:hover { background: var(--theme-color-background-hover, #eef1f5); }
57
+ .psd-btn:disabled { opacity: 0.5; pointer-events: none; }
58
+
59
+ .psd-report { display: flex; flex-wrap: wrap; gap: 0.5rem; }
60
+ .psd-chip { font-size: var(--theme-typography-size-sm, 0.82rem); padding: 0.25rem 0.6rem; border-radius: var(--theme-radius-pill, 999px);
61
+ background: var(--theme-color-background-tertiary, #eceef2); color: var(--theme-color-text-secondary, #45596b); }
62
+ .psd-chip strong { color: var(--theme-color-text-primary, #1f2733); }
63
+
64
+ .psd-banner { display: flex; align-items: center; gap: 0.45rem; padding: 0.55rem 0.75rem; border-radius: var(--theme-radius-md, 6px); font-size: var(--theme-typography-size-sm, 0.86rem); }
65
+ .psd-banner-ok { color: var(--theme-color-status-success, #2e7a3a); background: color-mix(in srgb, var(--theme-color-status-success, #2e7a3a) 12%, transparent); }
66
+ .psd-banner-warn { color: var(--theme-color-status-warning, #b8860b); background: color-mix(in srgb, var(--theme-color-status-warning, #d9a406) 14%, transparent); }
67
+ .psd-banner-error { color: var(--theme-color-status-error, #b62828); background: color-mix(in srgb, var(--theme-color-status-error, #b62828) 12%, transparent); }
68
+ .psd-banner-info { color: var(--theme-color-status-info, #1f6fb5); background: color-mix(in srgb, var(--theme-color-status-info, #1f6fb5) 12%, transparent); }
69
+
70
+ .psd-progress { height: 8px; border-radius: var(--theme-radius-pill, 999px); overflow: hidden; background: var(--theme-color-background-tertiary, #e9edf2); }
71
+ .psd-progress-fill { height: 100%; background: var(--theme-color-brand-primary, #156dd1); transition: width var(--theme-duration-normal, 0.2s) ease; }
72
+ .psd-row-actions { display: flex; gap: 0.5rem; flex-wrap: wrap; align-items: center; }
73
+ `;
package/source/Pict-Section-DataImport.js ADDED
@@ -0,0 +1,54 @@
1
+ // The container for all the Pict-Section-DataImport related code.
2
+ //
3
+ // pict-section-dataimport is an embeddable, configurable data-import wizard. A host registers the
4
+ // provider and calls createImportWizard(hash, config) to drop the whole flow into its UI:
5
+ // 1. Upload a CSV / TSV / Excel / fixed-width file (pict-section-upload).
6
+ // 2. Validate + detect its schema (columns, sample rows, fixed-width boundaries).
7
+ // 3. Map source columns to a target schema (a live Meadow entity schema, or a host-config schema).
8
+ // 4. Generate comprehensions (meadow-integration's browser-safe transform engine).
9
+ // 5. Push them — records via the browser EntityProvider, or one POST to a Comprehension endpoint.
10
+ // The step container is pict-section-accordion (wizard / accordion / stepper). All four extension
11
+ // points — Parser, Schema, Push, State — are host-swappable seams.
12
+
13
+ const PictProviderDataImport = require('./providers/Pict-Provider-DataImport.js');
14
+ const PictViewDataImportWizard = require('./views/PictView-DataImport-Wizard.js');
15
+
16
+ const DataImportComprehensionBuilder = require('./services/DataImport-ComprehensionBuilder.js');
17
+ const DataImportSession = require('./services/DataImport-Session.js');
18
+
19
+ const DataImportParserProvider = require('./seams/DataImport-ParserProvider.js');
20
+ const DataImportParserDelimited = require('./seams/DataImport-Parser-Delimited.js');
21
+ const DataImportParserFixedWidth = require('./seams/DataImport-Parser-FixedWidth.js');
22
+ const DataImportParserXlsx = require('./seams/DataImport-Parser-Xlsx.js');
23
+ const DataImportSchemaProvider = require('./seams/DataImport-SchemaProvider.js');
24
+ const DataImportPushTarget = require('./seams/DataImport-PushTarget.js');
25
+ const DataImportStateStore = require('./seams/DataImport-StateStore.js');
26
+
27
+ module.exports = PictProviderDataImport;
28
+
29
+ module.exports.PictProviderDataImport = PictProviderDataImport;
30
+ module.exports.PictViewDataImportWizard = PictViewDataImportWizard;
31
+
32
+ // Core services
33
+ module.exports.DataImportComprehensionBuilder = DataImportComprehensionBuilder;
34
+ module.exports.DataImportSession = DataImportSession;
35
+
36
+ // Seams (base classes + built-in adapters) — for hosts writing custom parsers / schema sources /
37
+ // push targets / state stores.
38
+ module.exports.ParserProvider = DataImportParserProvider;
39
+ module.exports.ParserDelimited = DataImportParserDelimited;
40
+ module.exports.ParserFixedWidth = DataImportParserFixedWidth;
41
+ module.exports.ParserXlsx = DataImportParserXlsx;
42
+ module.exports.SchemaProvider = DataImportSchemaProvider;
43
+ module.exports.SchemaProviderConfig = DataImportSchemaProvider.DataImportSchemaProviderConfig;
44
+ module.exports.PushTarget = DataImportPushTarget;
45
+ module.exports.PushTargetComprehension = DataImportPushTarget.DataImportPushTargetComprehension;
46
+ module.exports.StateStore = DataImportStateStore;
47
+ module.exports.StateStoreMemory = DataImportStateStore.DataImportStateStoreMemory;
48
+ module.exports.StateStoreLocal = DataImportStateStore.DataImportStateStoreLocal;
49
+
50
+ // The Meadow seam adapters are loaded lazily by the provider (they require the engine), but exposed
51
+ // here for direct use.
52
+ module.exports.getMeadowAdapters = () => require('./providers/Pict-Provider-DataImport-Meadow.js');
53
+
54
+ module.exports.default_configuration = PictProviderDataImport.default_configuration;
package/source/providers/Pict-Provider-DataImport-Meadow.js ADDED
@@ -0,0 +1,192 @@
1
+ // Pict-Provider-DataImport-Meadow — the Meadow-specific seam adapters: a SchemaProvider that reads
2
+ // entity schemas from a live Meadow server, and a PushTarget that lands comprehension records via the
3
+ // browser EntityProvider + meadow-integration's Integration Adapter (GUID marshaling + FK resolution).
4
+ //
5
+ // This file (plus the ComprehensionBuilder) is where the meadow-integration engine is touched; it is
6
+ // only required when SchemaSource:'meadow' or PushMode:'entityprovider' is used. The engine is reached
7
+ // through the browser-clean Meadow-Integration-Engine re-export (no server-only code).
8
+
9
+ const libEngine = require('meadow-integration/source/Meadow-Integration-Engine.js');
10
+ const libSchemaProvider = require('../seams/DataImport-SchemaProvider.js');
11
+ const libPushTarget = require('../seams/DataImport-PushTarget.js');
12
+
13
+ /**
14
+ * Wrap pict.EntityProvider as an Integration-Adapter REST client: adds getJSON (prefixed, for the
15
+ * schema fetch) + getEntityByGUID (via a filtered getEntitySet, used by the adapter's fallback path),
16
+ * and delegates upsert/get/delete straight through.
17
+ * @param {any} pPict @param {string} pURLPrefix @return {Record<string, any>}
18
+ */
19
+ const makeEntityProviderShim = (pPict, pURLPrefix) =>
20
+ {
21
+ const tmpEntityProvider = pPict.EntityProvider;
22
+ const tmpPrefix = pURLPrefix || (tmpEntityProvider.options && tmpEntityProvider.options.urlPrefix) || '/1.0/';
23
+ return {
24
+ serverURL: tmpPrefix,
25
+ restClient: tmpEntityProvider.restClient,
26
+ getJSON: (pURL, fCallback) =>
27
+ {
28
+ // The adapter passes an unprefixed 'Entity/Schema' — prepend the base URL unless already absolute.
29
+ const tmpFullURL = (pURL.indexOf('http') === 0 || pURL.indexOf(tmpPrefix) === 0) ? pURL : (tmpPrefix + pURL);
30
+ return tmpEntityProvider.restClient.getJSON(tmpFullURL, fCallback);
31
+ },
32
+ getEntity: (pEntity, pID, fCallback) => tmpEntityProvider.getEntity(pEntity, pID, fCallback),
33
+ upsertEntity: (pEntity, pRecord, fCallback) => tmpEntityProvider.upsertEntity(pEntity, pRecord, fCallback),
34
+ upsertEntities: (pEntity, pRecords, fCallback) => tmpEntityProvider.upsertEntities(pEntity, pRecords, fCallback),
35
+ deleteEntity: (pEntity, pID, fCallback) => tmpEntityProvider.deleteEntity(pEntity, pID, fCallback),
36
+ getEntityByGUID: (pEntity, pGUID, fCallback) =>
37
+ {
38
+ const tmpFilter = `FBV~GUID${pEntity}~EQ~${encodeURIComponent(pGUID)}`;
39
+ tmpEntityProvider.getEntitySet(pEntity, tmpFilter, (pError, pRecords) =>
40
+ {
41
+ if (pError) { return fCallback(pError); }
42
+ return fCallback(null, (Array.isArray(pRecords) && pRecords.length > 0) ? pRecords[0] : null);
43
+ });
44
+ },
45
+ };
46
+ };
47
+
48
+ /**
49
+ * SchemaProvider that reads each configured Meadow entity's schema from <URLPrefix><Entity>/Schema,
50
+ * mapping it to the dataimport schema shape (inferring foreign keys from ID<Other> columns).
51
+ */
52
+ class DataImportSchemaProviderMeadow extends libSchemaProvider
53
+ {
54
+ getSchema()
55
+ {
56
+ const tmpEntities = Array.isArray(this.options.MeadowEntities) ? this.options.MeadowEntities : [];
57
+ const tmpPrefix = this.options.URLPrefix || '/1.0/';
58
+ const tmpEntityProvider = this.pict.EntityProvider;
59
+ if (!tmpEntityProvider || !tmpEntityProvider.restClient)
60
+ {
61
+ return Promise.reject(new Error('pict-section-dataimport: a Meadow schema source needs pict.EntityProvider.'));
62
+ }
63
+
64
+ return Promise.all(tmpEntities.map((pEntityName) => new Promise((resolve) =>
65
+ {
66
+ tmpEntityProvider.restClient.getJSON(`${tmpPrefix}${pEntityName}/Schema`, (pError, pResponse, pParsedBody) =>
67
+ {
68
+ const tmpSchemaBody = (pParsedBody && typeof pParsedBody === 'object') ? pParsedBody
69
+ : ((pResponse && typeof pResponse === 'object') ? pResponse : {});
70
+ resolve(this._mapMeadowEntitySchema(pEntityName, tmpSchemaBody, tmpEntities));
71
+ });
72
+ }))).then((pEntityList) =>
73
+ {
74
+ return { Entities: pEntityList, Order: this._deriveOrder(pEntityList) };
75
+ });
76
+ }
77
+
78
+ /** Map a Meadow /Schema body to the { Entity, GUIDName, Fields } shape. */
79
+ _mapMeadowEntitySchema(pEntityName, pSchemaBody, pAllEntities)
80
+ {
81
+ const tmpGUIDName = `GUID${pEntityName}`;
82
+ let tmpColumns = [];
83
+ if (Array.isArray(pSchemaBody.Columns)) { tmpColumns = pSchemaBody.Columns; }
84
+ else if (pSchemaBody.Schema && Array.isArray(pSchemaBody.Schema)) { tmpColumns = pSchemaBody.Schema; }
85
+ else if (pSchemaBody.properties && typeof pSchemaBody.properties === 'object')
86
+ {
87
+ tmpColumns = Object.keys(pSchemaBody.properties).map((pKey) => Object.assign({ Column: pKey }, pSchemaBody.properties[pKey]));
88
+ }
89
+
90
+ // Audit / lifecycle columns Meadow stamps on every entity — noise in a column-mapping UI.
91
+ const tmpSystemColumns = { CreateDate: 1, CreatingIDUser: 1, UpdateDate: 1, UpdatingIDUser: 1, Deleted: 1, DeleteDate: 1, DeletingIDUser: 1 };
92
+ const tmpFields = tmpColumns.map((pColumn) =>
93
+ {
94
+ const tmpName = pColumn.Column || pColumn.Name || pColumn.column;
95
+ let tmpForeignKeyEntity = '';
96
+ if (tmpName && tmpName.indexOf('ID') === 0 && tmpName.length > 2)
97
+ {
98
+ const tmpReferenced = tmpName.slice(2);
99
+ if (pAllEntities.indexOf(tmpReferenced) >= 0 && tmpReferenced !== pEntityName) { tmpForeignKeyEntity = tmpReferenced; }
100
+ }
101
+ return {
102
+ Name: tmpName,
103
+ Type: pColumn.DataType || pColumn.Type || pColumn.type || 'string',
104
+ Size: Number(pColumn.Size || pColumn.size || 0) || 0,
105
+ Required: !!(pColumn.Required || pColumn.NonNull),
106
+ IsGUID: (tmpName === tmpGUIDName),
107
+ ForeignKeyEntity: tmpForeignKeyEntity,
108
+ };
109
+ }).filter((pField) => !!pField.Name
110
+ && pField.Name !== `ID${pEntityName}` // the auto-increment primary key
111
+ && !tmpSystemColumns[pField.Name]); // audit/lifecycle columns
112
+
113
+ return { Entity: pEntityName, GUIDName: tmpGUIDName, Fields: tmpFields };
114
+ }
115
+
116
+ /** Order entities referenced-before-referrer (entities with fewer outbound FKs first). */
117
+ _deriveOrder(pEntityList)
118
+ {
119
+ const tmpNames = pEntityList.map((pEntity) => pEntity.Entity);
120
+ const tmpFKCount = {};
121
+ pEntityList.forEach((pEntity) =>
122
+ {
123
+ tmpFKCount[pEntity.Entity] = pEntity.Fields.filter((pField) => pField.ForeignKeyEntity && tmpNames.indexOf(pField.ForeignKeyEntity) >= 0).length;
124
+ });
125
+ // Stable sort by outbound-FK count: zero-FK reference entities first, join tables last.
126
+ return tmpNames.slice().sort((pA, pB) => (tmpFKCount[pA] - tmpFKCount[pB]));
127
+ }
128
+ }
129
+
130
+ /**
131
+ * PushTarget that lands comprehension records via pict.EntityProvider, using a fresh Integration
132
+ * Adapter per entity (so GUID marshaling + FK resolution run through meadow-integration). Entities are
133
+ * pushed sequentially IN ORDER so each referenced entity's GUID->ID mappings populate the shared
134
+ * MeadowGUIDMap before a referrer marshals its foreign keys.
135
+ */
136
+ class DataImportPushTargetEntityProvider extends libPushTarget
137
+ {
138
+ push(pComprehension, pContext)
139
+ {
140
+ const tmpContext = pContext || {};
141
+ const tmpOrder = (Array.isArray(tmpContext.Order) && tmpContext.Order.length > 0) ? tmpContext.Order : Object.keys(pComprehension || {});
142
+ const tmpShim = makeEntityProviderShim(this.pict, tmpContext.ServerURL || tmpContext.URLPrefix);
143
+ const tmpPushed = [];
144
+ let tmpTotal = 0;
145
+ tmpOrder.forEach((pEntity) => { tmpTotal += Object.keys((pComprehension && pComprehension[pEntity]) || {}).length; });
146
+ let tmpDone = 0;
147
+
148
+ // Sequential reduce over the ordered entities (referenced entities first).
149
+ return tmpOrder.reduce((pPromise, pEntity) => pPromise.then(() =>
150
+ {
151
+ const tmpRecords = (pComprehension && pComprehension[pEntity]) || {};
152
+ const tmpGUIDs = Object.keys(tmpRecords);
153
+ if (tmpGUIDs.length === 0) { return; }
154
+
155
+ const tmpAdapterOptions = Object.assign({}, libEngine.MeadowIntegrationAdapter.default_configuration,
156
+ {
157
+ Entity: pEntity,
158
+ EntityGUIDMarshalPrefix: tmpContext.EntityGUIDPrefix || `E-${pEntity}`,
159
+ AdapterSetGUIDMarshalPrefix: tmpContext.GUIDPrefix || 'INTG-DEF',
160
+ SimpleMarshal: true,
161
+ ForceMarshal: true,
162
+ PerformDeletes: false,
163
+ AllowGUIDTruncation: !!tmpContext.AllowGUIDTruncation,
164
+ });
165
+ const tmpAdapter = new libEngine.MeadowIntegrationAdapter(this.pict, tmpAdapterOptions, `DataImport-${pEntity}`);
166
+ tmpAdapter.setRestClient(tmpShim);
167
+ tmpGUIDs.forEach((pGUID) => tmpAdapter.addSourceRecord(tmpRecords[pGUID]));
168
+
169
+ return new Promise((resolve, reject) =>
170
+ {
171
+ tmpAdapter.integrateRecords((pError) =>
172
+ {
173
+ if (pError) { return reject(pError); }
174
+ tmpPushed.push(pEntity);
175
+ tmpDone += tmpGUIDs.length;
176
+ if (typeof tmpContext.onProgress === 'function') { tmpContext.onProgress(tmpDone, tmpTotal); }
177
+ return resolve();
178
+ });
179
+ });
180
+ }), Promise.resolve()).then(() =>
181
+ {
182
+ return { Success: true, EntitiesPushed: tmpPushed, Message: `Pushed ${tmpDone} record(s) across ${tmpPushed.length} entity(ies) via EntityProvider.` };
183
+ });
184
+ }
185
+ }
186
+
187
+ module.exports =
188
+ {
189
+ makeEntityProviderShim,
190
+ DataImportSchemaProviderMeadow,
191
+ DataImportPushTargetEntityProvider,
192
+ };