pict-section-recordset 1.9.0 → 1.9.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pict-section-recordset",
-  "version": "1.9.0",
+  "version": "1.9.2",
   "description": "Pict dynamic record set management views",
   "main": "source/Pict-Section-RecordSet.js",
   "files": [
@@ -37,9 +37,9 @@
     "browser-env": "^3.3.0",
     "eslint": "^9.28.0",
     "jquery": "^3.7.1",
-    "pict": "^1.0.372",
+    "pict": "^1.0.374",
     "pict-application": "^1.0.34",
-    "pict-docuserve": "^1.4.4",
+    "pict-docuserve": "^1.4.19",
     "pict-service-commandlineutility": "^1.0.19",
     "quackage": "^1.3.0",
     "typescript": "^5.9.3"
@@ -48,7 +48,7 @@
     "fable-serviceproviderbase": "^3.0.19",
     "pict-provider": "^1.0.13",
     "pict-router": "^1.0.10",
-    "pict-section-form": "^1.0.196",
+    "pict-section-form": "^1.1.9",
     "pict-template": "^1.0.15",
     "pict-view": "^1.0.68",
     "sinon": "^21.0.1"

package/source/providers/RecordSet-RecordProvider-MeadowEndpoints.js

@@ -192,6 +192,63 @@ class MeadowEndpointsRecordSetProvider extends libRecordSetProviderBase
 		return [ tmpClauses, tmpExperience ];
 	}
 
+	/**
+	 * Derive the Lite `ExtraColumns` for a list fetch from the manifest's displayed
+	 * columns. Lite already returns the ID-prefixed, GUID-prefixed, CreatingIDUser and
+	 * UpdateDate fields plus a computed Value, so we only request the remaining scalar
+	 * display columns — and only ones that are real, non-blob schema columns. Returns
+	 * [] (caller then does a safe full fetch) if the manifest columns or schema are
+	 * unavailable.
+	 * @param {string} pEntity - The entity being listed.
+	 * @param {Record<string, any>} pOptions - The list options (carries RecordSetConfiguration).
+	 * @return {Array<string>} The ExtraColumns to request.
+	 */
+	_deriveLiteExtraColumns(pEntity, pOptions)
+	{
+		const tmpConfig = pOptions && pOptions.RecordSetConfiguration;
+		let tmpDescriptors = null;
+		if (tmpConfig && this.pict.PictSectionRecordSet && this.pict.PictSectionRecordSet.manifestDefinitions)
+		{
+			const tmpManifestHash = tmpConfig.RecordSetListDefaultManifest || (Array.isArray(tmpConfig.RecordSetListManifests) ? tmpConfig.RecordSetListManifests[0] : null);
+			const tmpManifest = tmpManifestHash ? this.pict.PictSectionRecordSet.manifestDefinitions[tmpManifestHash] : null;
+			tmpDescriptors = (tmpManifest && tmpManifest.Descriptors) || null;
+		}
+		const tmpSchemaColumns = (this._Schema && this._Schema.MeadowSchema && Array.isArray(this._Schema.MeadowSchema.Schema)) ? this._Schema.MeadowSchema.Schema : [];
+		if (!tmpDescriptors || tmpSchemaColumns.length < 1)
+		{
+			return [];
+		}
+		const tmpColumnType = {};
+		for (const tmpColumn of tmpSchemaColumns)
+		{
+			if (tmpColumn && tmpColumn.Column)
+			{
+				tmpColumnType[tmpColumn.Column] = tmpColumn.Type;
+			}
+		}
+		const tmpBlobTypes = { 'Text': true, 'JSON': true };
+		const tmpColumns = [];
+		for (const tmpKey of Object.keys(tmpDescriptors))
+		{
+			// ID*/GUID*/owner/update come back in every Lite record for free.
+			if (tmpKey.startsWith('ID') || tmpKey.startsWith('GUID') || tmpKey === 'CreatingIDUser' || tmpKey === 'UpdateDate')
+			{
+				continue;
+			}
+			// Only request real, non-blob columns (a computed/templated manifest column
+			// that is not a DB column would otherwise error the query).
+			if (!(tmpKey in tmpColumnType) || tmpBlobTypes[tmpColumnType[tmpKey]])
+			{
+				continue;
+			}
+			if (!tmpColumns.includes(tmpKey))
+			{
+				tmpColumns.push(tmpKey);
+			}
+		}
+		return tmpColumns;
+	}
+
 	/**
 	 * Read records from the provider.
 	 *
@@ -205,6 +262,30 @@ class MeadowEndpointsRecordSetProvider extends libRecordSetProviderBase
 			throw new Error('Entity is not defined in the provider options.');
 		}
 		const tmpEntity = pOptions.Entity || this.options.Entity;
+
+		// Lite projection (opt-in via RecordSetListLiteFetch): request only the columns
+		// the manifest displays so the list stops pulling blob columns (FormData, etc.).
+		// Lite (partial) records are NOT written to the entity cache (see getEntitySetPage) —
+		// the list renders them straight from state — so they can never poison the global
+		// cache that full-record consumers (row-click View, {~E:~}) rely on. The list's
+		// reference entities (Project/User) are full records and stay in the global cache,
+		// batched reliably by the connected-entity prefetch + the stale-read prune fix.
+		const tmpLiteFetch = (this.options.RecordSetListLiteFetch === true) || !!(pOptions && pOptions.RecordSetConfiguration && pOptions.RecordSetConfiguration.RecordSetListLiteFetch);
+		let tmpProjection = null;
+		if (tmpLiteFetch)
+		{
+			// Ensure the entity schema is loaded so we only request real, non-blob columns.
+			if (!this._Schema)
+			{
+				await this.getRecordSchema();
+			}
+			const tmpExtraColumns = this._deriveLiteExtraColumns(tmpEntity, pOptions);
+			if (tmpExtraColumns.length > 0)
+			{
+				tmpProjection = { Mode: 'LiteExtended', ExtraColumns: tmpExtraColumns };
+			}
+		}
+
 		if (this.pict.LogNoisiness > 1)
 		{
 			this.pict.log.info(`Reading ${tmpEntity} records`, { Options: pOptions });
@@ -212,6 +293,10 @@ class MeadowEndpointsRecordSetProvider extends libRecordSetProviderBase
 		return new Promise((resolve, reject) =>
 		{
 			const [ tmpClauses, tmpExperience ] = this._prepareFilterState(tmpEntity, pOptions);
+			if (tmpProjection)
+			{
+				tmpExperience.Projection = tmpProjection;
+			}
 			if (this.options.FilterEndpointOverride)
 			{
 				// Call the filtering endpoint with the clauses and experience.
@@ -237,7 +322,7 @@ class MeadowEndpointsRecordSetProvider extends libRecordSetProviderBase
 							}
 						}
 					}
-					this.pict.EntityProvider.cacheConnectedEntityRecordsWithoutCount(recordsReturn, IDFields, ['User', 'User'], false, () => 
+					this.pict.EntityProvider.cacheConnectedEntityRecordsWithoutCount(recordsReturn, IDFields, ['User', 'User'], false, () =>
 					{
 						resolve({ Records: recordsReturn, Facets: { } });
 					});
@@ -262,7 +347,12 @@ class MeadowEndpointsRecordSetProvider extends libRecordSetProviderBase
 						}
 					}
 				}
-				this.pict.EntityProvider.cacheConnectedEntityRecords(recordsReturn, IDFields, ['User', 'User'], false, () =>
+				// Use the NoCount (lazy-page) batch: counts are very costly in this MySQL,
+				// and the count-based variant serializes one slow COUNT per reference entity,
+				// stalling the queue so later entities (e.g. Project) never batch and fall
+				// back to per-row fetches. NoCount fetches each batch in one paged request
+				// with real concurrency (maxOperations).
+				this.pict.EntityProvider.cacheConnectedEntityRecordsWithoutCount(recordsReturn, IDFields, ['User', 'User'], false, () =>
 				{
 					resolve({ Records: recordsReturn, Facets: { } });
 				});
@@ -371,6 +461,11 @@ class MeadowEndpointsRecordSetProvider extends libRecordSetProviderBase
 				}
 				// A new record changes the total; drop the cached count so the next render re-counts.
 				this._RecordSetCountCache = null;
+				// Drop this list's scoped cache too, so the next render re-fetches fresh.
+				if (typeof this.pict.EntityProvider.clearScope === 'function')
+				{
+					this.pict.EntityProvider.clearScope(`RSList::${ this.options.RecordSet || this.options.Entity }`);
+				}
 				resolve(result);
 			});
 		});
@@ -400,6 +495,11 @@ class MeadowEndpointsRecordSetProvider extends libRecordSetProviderBase
 				}
 				// An edit can move a record in or out of the active filter; drop the cached count to be safe.
 				this._RecordSetCountCache = null;
+				// Drop this list's scoped cache too, so the next render re-fetches fresh.
+				if (typeof this.pict.EntityProvider.clearScope === 'function')
+				{
+					this.pict.EntityProvider.clearScope(`RSList::${ this.options.RecordSet || this.options.Entity }`);
+				}
 				resolve(result);
 			});
 		});
@@ -429,6 +529,11 @@ class MeadowEndpointsRecordSetProvider extends libRecordSetProviderBase
 				}
 				// A delete changes the total; drop the cached count so the next render re-counts.
 				this._RecordSetCountCache = null;
+				// Drop this list's scoped cache too, so the next render re-fetches fresh.
+				if (typeof this.pict.EntityProvider.clearScope === 'function')
+				{
+					this.pict.EntityProvider.clearScope(`RSList::${ this.options.RecordSet || this.options.Entity }`);
+				}
 				resolve(result);
 			});
 		});
@@ -535,11 +640,19 @@ class MeadowEndpointsRecordSetProvider extends libRecordSetProviderBase
 			const tmpFieldHumanName = this.getHumanReadableFieldName(pSchemaField);
 			const isUserAuditField = ['CreatingIDUser', 'DeletingIDUser', 'UpdatingIDUser'].includes(pSchemaField);
 			const customFilterClauses = this.options.Filters?.[pSchemaField];
-			if (pSchemaField.startsWith('ID') || pSchemaField.startsWith('ParentID') || isUserAuditField || customFilterClauses)
+			// The entity's own identity column (AutoIdentity / AutoGUID) — i.e. the primary key.
+			const isOwnIdentityField = pMeadowSchemaField && (pMeadowSchemaField.Type === 'AutoIdentity' || pMeadowSchemaField.Type === 'AutoGUID');
+			const isForeignKeyLike = pSchemaField.startsWith('ID') || pSchemaField.startsWith('ParentID') || isUserAuditField || customFilterClauses;
+			if (isForeignKeyLike)
 			{
 				for (const customField of Array.isArray(customFilterClauses) ? customFilterClauses : [customFilterClauses])
 				{
-					const remoteTableName = customField?.RemoteTable || pSchemaField.split('ID')[1];
+					// The table the picker pulls from: an explicit RemoteTable, else this
+					// recordset's declared Entity when the column is our own primary key (the PK
+					// references our own records), else the name peeled from the column for a
+					// plain foreign key. Peeling alone is the defect — a lake PK like
+					// `IDC182_HMA_MixDesign` peels to a table name that is not the entity name.
+					const remoteTableName = customField?.RemoteTable || (isOwnIdentityField ? this.options.Entity : pSchemaField.split('ID')[1]);
 					const fieldName = this.getHumanReadableFieldName(pSchemaField);
 					tmpFieldFilterClauses.push(Object.assign(
 					{
@@ -552,6 +665,7 @@ class MeadowEndpointsRecordSetProvider extends libRecordSetProviderBase
 						"EntityListEntryTemplate": this.getEntityListEntryTemplate(remoteTableName),
 						"CoreConnectionColumn": pSchemaField,
 						"RemoteTable": `${ remoteTableName }`,
+						"URLPrefix": this.options.URLPrefix,
 						"JoinExternalConnectionColumn": `ID${ remoteTableName }`,
 						"JoinInternalConnectionColumn": pSchemaField,
 						'DisplayName': `Selected Records`,
@@ -561,7 +675,10 @@ class MeadowEndpointsRecordSetProvider extends libRecordSetProviderBase
 					}, customField));
 				}
 			}
-			else
+			// The primary key is also a plain integer key, so it ALSO falls through to the
+			// Exact / In Range clauses below. A real foreign key gets only the picker; ordinary
+			// columns are matched by type.
+			if (!isForeignKeyLike || isOwnIdentityField)
 			{
 				switch (tmpFieldType)
 				{

package/source/views/filters/RecordSet-Filter-EntityReference-Base.js

@@ -233,6 +233,7 @@ class ViewRecordSetSUBSETFilterEntityReferenceBase extends ViewRecordSetSUBSETFi
 					Destination: pOffset > 0 ? `${this.getInformaryAddressPrefix()}${pClauseInformaryAddress}.SearchResultsAppend` : `${this.getInformaryAddressPrefix()}${pClauseInformaryAddress}.SearchResults`,
 					RecordStartCursor: pOffset,
 					PageSize: this.options.PageSize,
+					URLPrefix: tmpClause.URLPrefix,
 				}
 			],
 			() =>

package/source/views/list/RecordSet-List.js

@@ -31,7 +31,12 @@ const _DEFAULT_CONFIGURATION__List = (
 		AutoSolveWithApp: false,
 		AutoSolveOrdinal: 0,
 
-		CSS: false,
+		CSS: /*css*/`
+	.prsp-list-loading { display: flex; align-items: center; justify-content: center; min-height: 240px; width: 100%; }
+	.prsp-list-loading-inner { display: inline-flex; align-items: center; gap: 0.6em; color: var(--theme-color-text-muted, #64748b); font-size: 1.05rem; }
+	.prsp-list-spinner { display: inline-flex; animation: prsp-list-spin 0.9s linear infinite; }
+	@keyframes prsp-list-spin { from { transform: rotate(0deg); } to { transform: rotate(360deg); } }
+	`,
 		CSSPriority: 500,
 
 		Templates:
@@ -57,6 +62,17 @@ const _DEFAULT_CONFIGURATION__List = (
 					Hash: 'PRSP-List-Template-Record',
 					Template: /*html*/`
 	<!-- DefaultPackage end view template:  [PRSP-List-Template] -->
+	`
+				},
+				{
+					Hash: 'PRSP-List-LoadingShell',
+					Template: /*html*/`
+	<section id="PRSP_List_Loading" class="prsp-list-loading">
+		<div class="prsp-list-loading-inner">
+			<span class="prsp-list-spinner" aria-hidden="true">{~I:Refresh~}</span>
+			<span class="prsp-list-loading-label">Loading…</span>
+		</div>
+	</section>
 	`
 				}
 			],
@@ -138,6 +154,36 @@ class viewRecordSetList extends libPictRecordSetRecordView
 		return pRecordListData;
 	}
 
+	/**
+	 * Paint a loading shell into the list destination synchronously, before the data
+	 * fetch, so the previous page doesn't sit silently while a slow query runs. The
+	 * real list render (RenderMethod 'replace' into the same destination) overwrites
+	 * it when data arrives. Opt out with RecordSetListShowLoadingShell:false.
+	 * @param {Record<string, any>} pRecordListData
+	 */
+	_projectLoadingShell(pRecordListData)
+	{
+		try
+		{
+			const tmpConfig = pRecordListData && pRecordListData.RecordSetConfiguration;
+			if (tmpConfig && tmpConfig.RecordSetListShowLoadingShell === false)
+			{
+				return;
+			}
+			if (!pRecordListData || !pRecordListData.RenderDestination)
+			{
+				return;
+			}
+			this.pict.CSSMap.injectCSS();
+			this.pict.ContentAssignment.assignContent(pRecordListData.RenderDestination, this.pict.parseTemplateByHash('PRSP-List-LoadingShell', pRecordListData));
+		}
+		catch (pError)
+		{
+			// The loading shell is purely cosmetic; never let it break the list render.
+			this.log.warn(`RecordSetList: loading shell render failed: ${ pError && pError.message }`);
+		}
+	}
+
 	dynamicallyGenerateColumns(pRecordListData)
 	{
 		pRecordListData.TableCells = [];
@@ -240,6 +286,9 @@ class viewRecordSetList extends libPictRecordSetRecordView
 		};
 
 		// TODO: There are still problems with the way these have nested data.  Discuss how we might move that around
+		// Paint a loading shell before the (potentially slow) fetch so the prior page
+		// doesn't sit silently; the real list render replaces it when data arrives.
+		this._projectLoadingShell(tmpRecordListData);
 		// Fetch the records
 		const [ tmpRecords, tmpTotalRecordCount, tmpRecordSchema ] = await Promise.all([
 			this.pict.providers[pProviderHash].getRecords(tmpRecordListData),
@@ -504,6 +553,9 @@ class viewRecordSetList extends libPictRecordSetRecordView
 		};
 
 		// TODO: There are still problems with the way these have nested data.  Discuss how we might move that around
+		// Paint a loading shell before the (potentially slow) fetch so the prior page
+		// doesn't sit silently; the real list render replaces it when data arrives.
+		this._projectLoadingShell(tmpRecordListData);
 		// Fetch the records
 		tmpRecordListData.Records = await this.pict.providers[pProviderHash].getDecoratedRecords(tmpRecordListData);
 		// Get the total record count