pict-section-dataimport 1.0.0

package/source/seams/DataImport-SchemaProvider.js

@@ -0,0 +1,72 @@
+// DataImport-SchemaProvider — the abstract target-schema seam, plus the built-in config-schema
+// adapter (a complex, non-Meadow schema supplied entirely by host configuration).
+//
+// getSchema() returns the target entities + fields the mapping step maps columns ONTO:
+//   { Entities: [ { Entity, GUIDName, Fields: [ { Name, Type, Size, Required, IsGUID, ForeignKeyEntity,
+//     FKGUIDTemplate } ] } ], Order: [ entityName, ... ] }
+// The Meadow adapter (which fetches <URLPrefix><Entity>/Schema) lives in Pict-Provider-DataImport-Meadow.js
+// so this file — and the whole core — stays free of any Meadow/EntityProvider dependency.
+
+class DataImportSchemaProvider
+{
+	constructor(pPict, pOptions)
+	{
+		this.pict = pPict;
+		this.options = pOptions || {};
+	}
+
+	/**
+	 * @param {Record<string, any>} pContext
+	 * @return {Promise<{Entities:Array<any>, Order:Array<string>}>}
+	 */
+	getSchema(pContext)
+	{
+		return Promise.reject(new Error('pict-section-dataimport: SchemaProvider does not implement getSchema().'));
+	}
+
+	/** Normalize a raw schema descriptor: default GUIDName, ensure Fields arrays + an Order. @param {Record<string, any>} pSchema */
+	_normalizeSchema(pSchema)
+	{
+		const tmpEntities = (pSchema && Array.isArray(pSchema.Entities)) ? pSchema.Entities : [];
+		const tmpNormalized = tmpEntities.map((pEntity) =>
+		{
+			return {
+				Entity: pEntity.Entity,
+				GUIDName: pEntity.GUIDName || `GUID${pEntity.Entity}`,
+				GUIDTemplateDefault: pEntity.GUIDTemplateDefault || '',
+				Fields: Array.isArray(pEntity.Fields) ? pEntity.Fields.map((pField) =>
+				{
+					return Object.assign({
+						Name: pField.Name,
+						Type: pField.Type || 'string',
+						Size: (typeof pField.Size === 'number') ? pField.Size : 0,
+						Required: !!pField.Required,
+						IsGUID: (pField.IsGUID !== undefined) ? !!pField.IsGUID : (pField.Name === (pEntity.GUIDName || `GUID${pEntity.Entity}`)),
+						ForeignKeyEntity: pField.ForeignKeyEntity || '',
+						FKGUIDTemplate: pField.FKGUIDTemplate || '',
+					}, pField);
+				}) : [],
+			};
+		});
+		const tmpOrder = (pSchema && Array.isArray(pSchema.Order) && pSchema.Order.length > 0)
+			? pSchema.Order
+			: tmpNormalized.map((pEntity) => pEntity.Entity);
+		return { Entities: tmpNormalized, Order: tmpOrder };
+	}
+}
+
+/**
+ * Built-in adapter: the target schema is a descriptor passed directly in host config (the
+ * "complex REST schema defined by configuration" case). No network — just normalization.
+ */
+class DataImportSchemaProviderConfig extends DataImportSchemaProvider
+{
+	getSchema()
+	{
+		const tmpSchema = this.options.Schema || { Entities: [] };
+		return Promise.resolve(this._normalizeSchema(tmpSchema));
+	}
+}
+
+module.exports = DataImportSchemaProvider;
+module.exports.DataImportSchemaProviderConfig = DataImportSchemaProviderConfig;

package/source/seams/DataImport-StateStore.js

@@ -0,0 +1,123 @@
+// DataImport-StateStore — the abstract persistence seam, plus built-in memory + localStorage adapters.
+//
+// This is what lets a user reload a partially-mapped dataset: the wizard serializes its session (file
+// ref, parse config, detected columns, the mapping) and save()s it; load()/list() bring it back. A
+// host can supply a custom store (or the lighter PersistenceHook on the wizard config) to persist
+// sessions on a server instead.
+
+class DataImportStateStore
+{
+	constructor(pPict, pOptions)
+	{
+		this.pict = pPict;
+		this.options = pOptions || {};
+	}
+
+	/** @param {string} pSessionId @param {Record<string, any>} pSerializedSession @return {Promise<void>} */
+	save(pSessionId, pSerializedSession) { return Promise.reject(new Error('not implemented')); }
+	/** @param {string} pSessionId @return {Promise<Record<string, any>|null>} */
+	load(pSessionId) { return Promise.reject(new Error('not implemented')); }
+	/** @return {Promise<Array<{SessionId:string, Title:string, UpdatedAt:number}>>} */
+	list() { return Promise.reject(new Error('not implemented')); }
+	/** @param {string} pSessionId @return {Promise<void>} */
+	remove(pSessionId) { return Promise.reject(new Error('not implemented')); }
+}
+
+/** Ephemeral in-process store (fine for tests + transient sessions). */
+class DataImportStateStoreMemory extends DataImportStateStore
+{
+	constructor(pPict, pOptions)
+	{
+		super(pPict, pOptions);
+		this._map = {};
+	}
+
+	save(pSessionId, pSerializedSession)
+	{
+		this._map[pSessionId] = JSON.parse(JSON.stringify(pSerializedSession));
+		return Promise.resolve();
+	}
+	load(pSessionId)
+	{
+		return Promise.resolve(this._map[pSessionId] ? JSON.parse(JSON.stringify(this._map[pSessionId])) : null);
+	}
+	list()
+	{
+		return Promise.resolve(Object.keys(this._map).map((pKey) =>
+		{
+			const tmpSession = this._map[pKey] || {};
+			return { SessionId: pKey, Title: tmpSession.Title || pKey, UpdatedAt: tmpSession.UpdatedAt || 0 };
+		}));
+	}
+	remove(pSessionId)
+	{
+		delete this._map[pSessionId];
+		return Promise.resolve();
+	}
+}
+
+/** Browser localStorage store, keyed by an options.KeyPrefix (default 'dataimport:'). */
+class DataImportStateStoreLocal extends DataImportStateStore
+{
+	get _prefix() { return this.options.KeyPrefix || 'dataimport:'; }
+
+	_storage()
+	{
+		if (typeof localStorage === 'undefined') { throw new Error('pict-section-dataimport: localStorage is not available in this environment.'); }
+		return localStorage;
+	}
+
+	save(pSessionId, pSerializedSession)
+	{
+		try { this._storage().setItem(this._prefix + pSessionId, JSON.stringify(pSerializedSession)); return Promise.resolve(); }
+		catch (pError) { return Promise.reject(pError); }
+	}
+	load(pSessionId)
+	{
+		try
+		{
+			const tmpRaw = this._storage().getItem(this._prefix + pSessionId);
+			return Promise.resolve(tmpRaw ? JSON.parse(tmpRaw) : null);
+		}
+		catch (pError) { return Promise.reject(pError); }
+	}
+	list()
+	{
+		try
+		{
+			const tmpStorage = this._storage();
+			const tmpSessions = [];
+			for (let i = 0; i < tmpStorage.length; i++)
+			{
+				const tmpKey = tmpStorage.key(i);
+				if (tmpKey && tmpKey.indexOf(this._prefix) === 0)
+				{
+					let tmpSession = {};
+					try { tmpSession = JSON.parse(tmpStorage.getItem(tmpKey)) || {}; } catch (pError) { tmpSession = {}; }
+					tmpSessions.push({ SessionId: tmpKey.slice(this._prefix.length), Title: tmpSession.Title || tmpKey, UpdatedAt: tmpSession.UpdatedAt || 0 });
+				}
+			}
+			return Promise.resolve(tmpSessions);
+		}
+		catch (pError) { return Promise.reject(pError); }
+	}
+	remove(pSessionId)
+	{
+		try { this._storage().removeItem(this._prefix + pSessionId); return Promise.resolve(); }
+		catch (pError) { return Promise.reject(pError); }
+	}
+}
+
+/** Adapter that wraps a host-supplied PersistenceHook { save, load, list, remove } (e.g. server persistence). */
+class DataImportStateStoreHook extends DataImportStateStore
+{
+	save(pSessionId, pSerializedSession) { return Promise.resolve(this.options.Hook.save(pSessionId, pSerializedSession)); }
+	load(pSessionId) { return Promise.resolve(this.options.Hook.load(pSessionId)); }
+	list() { return Promise.resolve(this.options.Hook.list ? this.options.Hook.list() : []); }
+	remove(pSessionId) { return Promise.resolve(this.options.Hook.remove ? this.options.Hook.remove(pSessionId) : undefined); }
+}
+
+module.exports = DataImportStateStore;
+module.exports.DataImportStateStoreMemory = DataImportStateStoreMemory;
+module.exports.DataImportStateStoreLocal = DataImportStateStoreLocal;
+module.exports.DataImportStateStoreHook = DataImportStateStoreHook;

package/source/services/DataImport-ComprehensionBuilder.js

@@ -0,0 +1,123 @@
+// DataImport-ComprehensionBuilder — turns a mapping + parsed rows into a Comprehension, using
+// meadow-integration's browser-safe transform engine (reached via the clean Meadow-Integration-Engine
+// re-export, which pulls in NO server-only code — verified). This is the only client-side piece that
+// needs the engine, and it's the heart of the module: it runs the canonical transformRecord pipeline
+// (Solvers + MultipleGUIDUniqueness fan-out) so the comprehensions are byte-identical to what the
+// meadow-integration CLI would produce from the same mapping files.
+
+const libEngine = require('meadow-integration/source/Meadow-Integration-Engine.js');
+
+class DataImportComprehensionBuilder
+{
+	/**
+	 * @param {any} pPict
+	 * @param {any} [pTransform] - inject a transform engine (tests); else one is built from the engine export.
+	 */
+	constructor(pPict, pTransform)
+	{
+		this.pict = pPict;
+		this.transform = pTransform || new libEngine.MeadowIntegrationTabularTransform(pPict);
+	}
+
+	/**
+	 * Build a comprehension from a multi-entity mapping + parsed rows.
+	 * @param {Record<string, any>} pMapping - { Order:[entityName,…], Entities:{ <name>: <entityMapping> } }
+	 *   where each entityMapping is the meadow mapping shape: { Entity, GUIDName?, GUIDTemplate, Mappings,
+	 *   Solvers?, MultipleGUIDUniqueness?, ManyfestAddresses? }.
+	 * @param {Array<Record<string, any>>} pRows
+	 * @return {{Comprehension:Record<string, any>, Report:Record<string, any>}}
+	 */
+	build(pMapping, pRows)
+	{
+		const tmpRows = Array.isArray(pRows) ? pRows : [];
+		const tmpEntities = (pMapping && pMapping.Entities) || {};
+		const tmpOrder = (pMapping && Array.isArray(pMapping.Order) && pMapping.Order.length > 0)
+			? pMapping.Order
+			: Object.keys(tmpEntities);
+
+		const tmpComprehension = {};
+		const tmpReport = { ParsedRowCount: tmpRows.length, EntityCounts: {}, BadRecords: [] };
+
+		// Process entities IN ORDER so the comprehension's keys come out referenced-before-referrer
+		// (the push paths do no topo-sort — emission order is load-bearing for FK resolution).
+		for (let e = 0; e < tmpOrder.length; e++)
+		{
+			const tmpEntityKey = tmpOrder[e];
+			const tmpEntityMapping = tmpEntities[tmpEntityKey];
+			if (!tmpEntityMapping || !tmpEntityMapping.Entity) { continue; }
+
+			const tmpOutcome = this.transform.newMappingOutcomeObject();
+			// Set a truthy ImplicitConfiguration so the engine's implicit-generation branch (which would
+			// otherwise dereference an undefined variable) never runs; our explicit mapping is authoritative.
+			tmpOutcome.ImplicitConfiguration = {};
+			tmpOutcome.ExplicitConfiguration = tmpEntityMapping;
+			this.transform.initializeMappingOutcomeObject(tmpOutcome);
+
+			for (let r = 0; r < tmpRows.length; r++)
+			{
+				try { this.transform.transformRecord(tmpRows[r], tmpOutcome); }
+				catch (pError)
+				{
+					tmpReport.BadRecords.push({ Entity: tmpEntityMapping.Entity, RowIndex: r, Reason: pError.message || String(pError), Row: tmpRows[r] });
+				}
+			}
+
+			const tmpEntityName = tmpEntityMapping.Entity;
+			tmpComprehension[tmpEntityName] = tmpOutcome.Comprehension[tmpEntityName] || {};
+			tmpReport.EntityCounts[tmpEntityName] = Object.keys(tmpComprehension[tmpEntityName]).length;
+			(tmpOutcome.BadRecords || []).forEach((pBadRow) =>
+			{
+				tmpReport.BadRecords.push({ Entity: tmpEntityName, Reason: 'No valid GUID generated for record', Row: pBadRow });
+			});
+		}
+
+		return { Comprehension: tmpComprehension, Report: tmpReport };
+	}
+
+	/**
+	 * Pre-flight FK-ordering check: flag any `GUID<Other>` mapping field whose referenced entity appears
+	 * LATER than (or not in) the Order — those FKs would silently resolve to NULL/0 at push time.
+	 * @param {Record<string, any>} pMapping
+	 * @return {Array<{Entity:string, Field:string, ReferencedEntity:string, Message:string}>}
+	 */
+	validateForeignKeyOrder(pMapping)
+	{
+		const tmpEntities = (pMapping && pMapping.Entities) || {};
+		const tmpOrder = (pMapping && Array.isArray(pMapping.Order) && pMapping.Order.length > 0)
+			? pMapping.Order
+			: Object.keys(tmpEntities);
+		const tmpEntityNames = tmpOrder
+			.map((pKey) => tmpEntities[pKey] && tmpEntities[pKey].Entity)
+			.filter(Boolean);
+		const tmpIndexByEntity = {};
+		tmpEntityNames.forEach((pName, pIndex) => { tmpIndexByEntity[pName] = pIndex; });
+
+		const tmpWarnings = [];
+		tmpOrder.forEach((pKey, pIndex) =>
+		{
+			const tmpEntityMapping = tmpEntities[pKey];
+			if (!tmpEntityMapping || !tmpEntityMapping.Mappings) { return; }
+			const tmpGUIDName = tmpEntityMapping.GUIDName || `GUID${tmpEntityMapping.Entity}`;
+			Object.keys(tmpEntityMapping.Mappings).forEach((pFieldName) =>
+			{
+				if (pFieldName === tmpGUIDName) { return; }                 // the entity's own GUID, not an FK
+				if (pFieldName.indexOf('GUID') !== 0) { return; }            // only GUID<Other> fields are FKs
+				const tmpReferenced = pFieldName.slice(4);
+				if (!tmpReferenced || tmpReferenced === tmpEntityMapping.Entity) { return; }
+				if (!(tmpReferenced in tmpIndexByEntity)) { return; }        // references something outside this import — fine
+				if (tmpIndexByEntity[tmpReferenced] >= pIndex)
+				{
+					tmpWarnings.push({
+						Entity: tmpEntityMapping.Entity,
+						Field: pFieldName,
+						ReferencedEntity: tmpReferenced,
+						Message: `"${tmpEntityMapping.Entity}.${pFieldName}" references "${tmpReferenced}", which is generated after it — reorder so "${tmpReferenced}" comes first, or its foreign key will be unresolved.`,
+					});
+				}
+			});
+		});
+		return tmpWarnings;
+	}
+}
+
+module.exports = DataImportComprehensionBuilder;

package/source/services/DataImport-Session.js

@@ -0,0 +1,101 @@
+// DataImport-Session — the per-import-session state shape + (de)serialization helpers.
+//
+// One session lives at AppData.DataImport.Sessions[<wizardHash>] and is fully JSON-serializable: the
+// live uploaded file handle is referenced indirectly (File.Ref), never embedded, so a StateStore can
+// persist a partially-mapped session and reload it later. `Mapping` is the durable artifact;
+// `Comprehension` is a recomputable cache (re-derivable from Mapping + re-parsed rows).
+
+/** @return {number} A timestamp (plain module code — Date.now is fine here). */
+const now = () => Date.now();
+
+/**
+ * Build a fresh import session.
+ * @param {string} pSessionId @param {Record<string, any>} pConfig
+ * @return {Record<string, any>}
+ */
+const newImportSession = (pSessionId, pConfig) =>
+{
+	const tmpConfig = pConfig || {};
+	const tmpTimestamp = now();
+	return {
+		SessionId: pSessionId,
+		Title: tmpConfig.Title || pSessionId,
+		SchemaVersion: 1,
+		CreatedAt: tmpTimestamp,
+		UpdatedAt: tmpTimestamp,
+
+		CurrentStep: 'upload',     // upload | detect | mapping | generate | push
+		StepStatus: { upload: 'active', detect: 'pending', mapping: 'pending', generate: 'pending', push: 'pending' },
+
+		// File REF (not the live handle/bytes): { Ref, Name, Size, Type, Kind, Stored, StorageRef }.
+		File: null,
+
+		ParseConfig:
+		{
+			Kind: null,            // csv | tsv | fixedwidth | xlsx
+			Delimited: { Delimiter: ',', HasHeader: true },
+			FixedWidth: { Columns: [], HasHeader: false },
+			Xlsx: { HasHeader: true },
+			SampleRowLimit: 25,
+		},
+		DetectedColumns: [],
+		SampleRows: [],
+		RawLines: [],              // fixed-width ruler source (when no column spec yet)
+		RowCountEstimate: 0,
+
+		Mapping:
+		{
+			TargetSchemaSource: tmpConfig.SchemaSource || 'meadow',
+			Order: [],
+			Entities: {},          // <entityName>: { Entity, GUIDName, GUIDTemplate, Mappings, Solvers, MultipleGUIDUniqueness, _ColumnBindings }
+		},
+
+		Comprehension: null,       // recomputable cache: { <Entity>: { <guid>: record } }
+		GenerationReport: null,    // { ParsedRowCount, EntityCounts, BadRecords }
+
+		Push:
+		{
+			Mode: tmpConfig.PushMode || 'entityprovider',
+			Status: 'idle',        // idle | running | complete | error
+			Progress: { Total: 0, Pushed: 0 },
+			Result: null,
+			Error: null,
+		},
+	};
+};
+
+/**
+ * A JSON-safe snapshot for persistence. The Comprehension can be huge + is recomputable, so it is
+ * dropped by default (pass pIncludeComprehension to keep it).
+ * @param {Record<string, any>} pSession @param {boolean} [pIncludeComprehension]
+ * @return {Record<string, any>}
+ */
+const serialize = (pSession, pIncludeComprehension) =>
+{
+	const tmpSnapshot = JSON.parse(JSON.stringify(pSession || {}));
+	if (!pIncludeComprehension) { tmpSnapshot.Comprehension = null; }
+	tmpSnapshot.UpdatedAt = now();
+	return tmpSnapshot;
+};
+
+/**
+ * Rebuild a session object from a snapshot (filling any new fields a newer SchemaVersion added).
+ * @param {Record<string, any>} pSnapshot
+ * @return {Record<string, any>}
+ */
+const hydrate = (pSnapshot) =>
+{
+	const tmpSnapshot = pSnapshot || {};
+	const tmpBase = newImportSession(tmpSnapshot.SessionId || 'import', {});
+	// Deep-merge the saved fields over the fresh base (saved values win; new base fields survive).
+	return Object.assign(tmpBase, tmpSnapshot, {
+		ParseConfig: Object.assign({}, tmpBase.ParseConfig, tmpSnapshot.ParseConfig),
+		Mapping: Object.assign({}, tmpBase.Mapping, tmpSnapshot.Mapping),
+		Push: Object.assign({}, tmpBase.Push, tmpSnapshot.Push),
+	});
+};
+
+/** Mark a session's UpdatedAt. @param {Record<string, any>} pSession */
+const touch = (pSession) => { if (pSession) { pSession.UpdatedAt = now(); } return pSession; };
+
+module.exports = { newImportSession, serialize, hydrate, touch };