retold-data-service 2.0.43 → 2.1.1

package/.github/workflows/publish-image.yml

@@ -0,0 +1,66 @@
+# Publish a container image to GitHub Container Registry on every
+# version tag push (e.g. `v1.2.3`). Generated by `quack docker-init`.
+#
+# Image lands at:
+#   ghcr.io/stevenvelozo/retold-data-service:<version>
+#   ghcr.io/stevenvelozo/retold-data-service:latest   (only on stable tags)
+
+name: Publish container image
+
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Tag to apply (e.g. dev or 1.2.3-test). `latest` is reserved for stable tag pushes.'
+        required: true
+        default: 'dev'
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU (multi-arch support)
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Compute image tags
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/${{ github.repository_owner }}/retold-data-service
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=raw,value=${{ github.event.inputs.tag }},enable=${{ github.event_name == 'workflow_dispatch' }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

package/BUILDING-AND-PUBLISHING.md

@@ -0,0 +1,140 @@
+# Building and Publishing
+
+How to ship `retold-data-service` to npm and to GitHub Container Registry
+(GHCR). Generated by `quack docker-init`; the structure matches the
+shared template across all dockerized retold tools.
+
+`retold-data-service` is a **long-running service**. Restart policy in compose / k8s should be `unless-stopped` (or equivalent). The Dockerfile should declare a HEALTHCHECK against the service's health endpoint.
+
+---
+
+## TL;DR
+
+```bash
+# npm-only release (the default — most common case)
+npm run release:patch
+
+# npm release that ALSO rebuilds the GHCR image
+npm run release:patch:image
+```
+
+The default release is npm-only. Docker images are deliberate, opt-in
+artifacts because each multi-arch build burns several minutes of CI
+time. Use `:image` (or set `BUILD_DOCKER=1`) when runtime code,
+dependencies, env-var contract, or the Dockerfile changed.
+
+---
+
+## Prerequisites (one-time setup)
+
+- **npm login** — `npm whoami` should print your username.
+- **Git remote configured** — `git remote get-url origin` should print
+  `git@github.com:stevenvelozo/retold-data-service.git` (or the
+  HTTPS equivalent).
+- **Push access to the repo** — required so `postversion` /
+  `postpublish` hooks can push commits and tags.
+- **Docker** (only if you want to test the image locally before tag).
+
+---
+
+## Ecosystem convention: lockfiles are gitignored
+
+`package-lock.json` is in this repo's `.gitignore` (Quackage convention
+shared across the retold ecosystem). The Dockerfile must use `npm install`,
+not `npm ci` — `npm ci` requires the lockfile to be in the build context
+and CI runners only check out what's in git. If you see EUSAGE errors in
+GHCR build logs, change `RUN npm ci` to `RUN npm install` in the
+Dockerfile.
+
+---
+
+## Releasing
+
+| Command                              | npm registry | GHCR image rebuild |
+|--------------------------------------|--------------|--------------------|
+| `npm run release:patch`              | yes          | no                 |
+| `npm run release:patch:image`        | yes          | yes                |
+| `npm run release:minor`              | yes          | no                 |
+| `npm run release:minor:image`        | yes          | yes                |
+| `npm run release:major`              | yes          | no                 |
+| `npm run release:major:image`        | yes          | yes                |
+
+The non-`:image` variants are the default because most patch releases
+don't change runtime behavior. The `:image` variants tell the pipeline
+"this release does change runtime — build me a new image."
+
+### Direct CLI (also works)
+
+```bash
+npm publish                              # npm only
+npm run publish:docker                   # npm + docker (sets BUILD_DOCKER=1)
+```
+
+### From `retold-manager` TUI
+
+- `[!]` Publish — npm only
+- `[D]` Publish with docker image — npm + GHCR build
+
+### Promoting a previous npm release to docker later
+
+If you released `v<x>` to npm only, then later decide you do want a
+docker image:
+
+```bash
+git push origin v<x>    # pushes the local tag → GHCR fires
+```
+
+The local tag is still sitting there from the original `npm version`
+step. Pushing it triggers the workflow without touching npm.
+
+---
+
+## The chain
+
+The lifecycle hooks all live in `package.json` and delegate to
+`npx quack release …`. Default path (`BUILD_DOCKER` unset):
+
+```
+npm publish
+  → prepublishOnly: npm test                     ← test gate
+  → publish to npm
+  → postpublish: BUILD_DOCKER unset → no-op      ← image NOT triggered
+```
+
+Docker-included path (`BUILD_DOCKER=1`):
+
+```
+BUILD_DOCKER=1 npm publish    (or: npm run publish:docker)
+  → prepublishOnly: npm test
+  → publish to npm
+  → postpublish: BUILD_DOCKER=1 → tag + push     ← image trigger
+  → .github/workflows/publish-image.yml fires
+  → docker buildx build linux/amd64,linux/arm64
+  → docker push ghcr.io/stevenvelozo/retold-data-service:<version>
+```
+
+---
+
+## Verifying a release
+
+1. **npm**: `npm view retold-data-service version`
+2. **Workflow**: `https://github.com/stevenvelozo/retold-data-service/actions`
+3. **Image**: `docker pull ghcr.io/stevenvelozo/retold-data-service:latest`
+
+If the first `docker pull` returns `denied`, the package is private by
+default — flip visibility to public via Package Settings → Danger Zone
+on the package page.
+
+---
+
+## Image consumption
+
+```bash
+docker pull ghcr.io/stevenvelozo/retold-data-service:latest
+docker run --rm ghcr.io/stevenvelozo/retold-data-service:latest
+```
+
+Configuration via env vars: see this module's README for the supported
+`<MODULE>_*` variables. Any secret-bearing var also accepts `<NAME>_FILE`
+pointing at a file whose contents become the value (mysql/postgres
+convention; works with docker secrets and k8s Secrets).

package/Dockerfile

@@ -0,0 +1,21 @@
+FROM node:20-bookworm AS base
+MAINTAINER steven velozo <steven@velozo.com>
+
+RUN apt-get update && apt-get -y --force-yes install curl vim nano less \
+	tmux uuid-runtime
+
+ADD package.json /service_root/package.json
+RUN cd /service_root && npm install --omit=dev
+
+ADD source /service_root/source
+ADD bin    /service_root/bin
+
+WORKDIR /service_root
+
+RUN rm -rf package-lock.json .git test
+
+FROM base AS production
+
+RUN date -u +"%Y-%m-%dT%H:%M:%SZ" > ./build.date
+
+CMD ["node", "bin/retold-data-service-clone.js"]

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "retold-data-service",
-    "version": "2.0.43",
+    "version": "2.1.1",
     "description": "Serve up a whole model!",
     "main": "source/Retold-Data-Service.js",
     "bin": {
@@ -22,7 +22,16 @@
         "docker-service-shell": "docker exec -it retold-data-service /bin/bash",
         "test:integration": "node test/run-integration-tests.js",
         "test:integration:no-browser": "node test/run-integration-tests.js --skip-puppeteer",
-        "test:all": "npx quack test && node test/run-integration-tests.js --skip-puppeteer"
+        "test:all": "npx quack test && node test/run-integration-tests.js --skip-puppeteer",
+        "postversion": "npx quack release postversion",
+        "postpublish": "npx quack release postpublish",
+        "publish:docker": "npx quack release publish --image",
+        "release:patch": "npx quack release patch",
+        "release:minor": "npx quack release minor",
+        "release:major": "npx quack release major",
+        "release:patch:image": "npx quack release patch --image",
+        "release:minor:image": "npx quack release minor --image",
+        "release:major:image": "npx quack release major --image"
     },
     "mocha": {
         "spec": "test/RetoldDataService_tests.js",
@@ -59,27 +68,29 @@
     },
     "homepage": "https://github.com/stevenvelozo/retold-data-service",
     "devDependencies": {
-        "meadow-connection-sqlite": "^1.0.18",
+        "meadow-connection-sqlite": "^1.0.19",
         "pict-docuserve": "^0.1.5",
         "puppeteer": "^24.40.0",
-        "quackage": "^1.1.1",
+        "quackage": "^1.2.0",
         "stricture": "^4.0.2",
         "supertest": "^7.2.2"
     },
     "dependencies": {
-        "bibliograph": "^0.1.4",
-        "fable": "^3.1.70",
+        "bibliograph": "^0.1.6",
+        "fable": "^3.1.71",
         "fable-serviceproviderbase": "^3.0.19",
-        "meadow": "^2.0.35",
-        "meadow-connection-mysql": "^1.0.17",
-        "meadow-endpoints": "^4.0.15",
-        "meadow-integration": "^1.0.35",
-        "meadow-migrationmanager": "^0.0.13",
-        "orator": "^6.0.4",
+        "meadow": "^2.0.37",
+        "meadow-connection-manager": "^1.1.0",
+        "pict-section-connection-form": "^0.0.1",
+        "meadow-connection-mysql": "^1.0.18",
+        "meadow-endpoints": "^4.0.17",
+        "meadow-integration": "^1.0.38",
+        "meadow-migrationmanager": "^0.0.14",
+        "orator": "^6.1.1",
         "orator-http-proxy": "^1.0.5",
         "orator-serviceserver-restify": "^2.0.10",
-        "orator-static-server": "^2.0.4",
-        "pict": "^1.0.363",
+        "orator-static-server": "^2.1.3",
+        "pict": "^1.0.365",
         "pict-section-histogram": "^1.0.0",
         "pict-sessionmanager": "^1.0.2",
         "stricture": "^4.0.2"

package/source/services/data-cloner/DataCloner-Command-Connection.js

@@ -2,17 +2,57 @@
  * DataCloner Connection Management Routes
  *
  * Registers /clone/connection/* endpoints for managing the local database
- * connection (status, configure, test).
+ * connection (status, configure, test, schemas).
  *
  * @param {Object} pDataClonerService - The RetoldDataServiceDataCloner instance
  * @param {Object} pOratorServiceServer - The Orator ServiceServer instance
  */
+const libMeadowConnectionManager = require('meadow-connection-manager');
+
 module.exports = (pDataClonerService, pOratorServiceServer) =>
 {
 	let tmpFable = pDataClonerService.fable;
 	let tmpCloneState = pDataClonerService.cloneState;
 	let tmpPrefix = pDataClonerService.routePrefix;
 
+	// MCM is the canonical aggregator for connection-form schemas; the
+	// DataCloner browser UI fetches /clone/connection/schemas and renders
+	// its provider form straight off the result, instead of carrying
+	// per-provider HTML in the bundle.  Cached on the service so we don't
+	// re-walk the filesystem on every poll.
+	let tmpSchemaMCM = null;
+	let tmpCachedSchemas = null;
+	let tmpGetSchemas = () =>
+	{
+		if (tmpCachedSchemas) { return tmpCachedSchemas; }
+		if (!tmpSchemaMCM)
+		{
+			tmpSchemaMCM = new libMeadowConnectionManager(tmpFable, {}, 'datacloner-mcm-formschemas');
+		}
+		// Defensive: an older MCM (< 1.1.0) won't have the aggregator
+		// method.  Surface an empty array rather than throwing so the
+		// UI can fall back to a "schemas unavailable" message.
+		if (typeof(tmpSchemaMCM.getAllProviderFormSchemas) !== 'function')
+		{
+			tmpFable.log.warn('DataCloner: meadow-connection-manager is older than 1.1.0; connection form schemas are not available.  Run npm update meadow-connection-manager.');
+			tmpCachedSchemas = [];
+			return tmpCachedSchemas;
+		}
+		tmpCachedSchemas = tmpSchemaMCM.getAllProviderFormSchemas();
+		return tmpCachedSchemas;
+	};
+
+	// GET /clone/connection/schemas
+	// Returns the form schemas for every provider whose module is
+	// installed in the host environment.  Drives the connection
+	// section's UI (provider picker + per-provider field block).
+	pOratorServiceServer.get(`${tmpPrefix}/connection/schemas`,
+		(pRequest, pResponse, fNext) =>
+		{
+			pResponse.send(200, { Schemas: tmpGetSchemas() });
+			return fNext();
+		});
+
 	// GET /clone/connection/status
 	pOratorServiceServer.get(`${tmpPrefix}/connection/status`,
 		(pRequest, pResponse, fNext) =>

package/source/services/data-cloner/pict-app/Pict-Application-DataCloner.js

@@ -11,6 +11,7 @@ const libViewSync = require('./views/PictView-DataCloner-Sync.js');
 const libViewExport = require('./views/PictView-DataCloner-Export.js');
 const libViewViewData = require('./views/PictView-DataCloner-ViewData.js');
 const libViewHistogram = require('pict-section-histogram');
+const libViewConnectionForm = require('pict-section-connection-form');
 
 class DataClonerApplication extends libPictApplication
 {
@@ -45,11 +46,35 @@ class DataClonerApplication extends libPictApplication
 				BarColor: '#4a90d9',
 				Bins: []
 			}, libViewHistogram);
+
+		// Shared schema-driven connection form.  Renders into the slot
+		// the DataCloner-Connection accordion shell exposes; the
+		// provider's bootstrapConnectionSchemas() pumps the schemas in
+		// once the host's /clone/connection/schemas endpoint responds.
+		this.pict.addView('PictSection-ConnectionForm',
+			Object.assign({}, libViewConnectionForm.default_configuration,
+				{
+					ContainerSelector:         '#DataCloner-Connection-FormSlot',
+					DefaultDestinationAddress: '#DataCloner-Connection-FormSlot',
+					SchemasAddress:            'AppData.DataCloner.Connection.Schemas',
+					ActiveAddress:             'AppData.DataCloner.Connection.ActiveProvider',
+					FieldIDPrefix:             'datacloner-conn'
+				}), libViewConnectionForm);
 	}
 
 	onAfterInitializeAsync(fCallback)
 	{
-		// Centralized state (replaces global variables)
+		// Centralized state (replaces global variables).
+		//
+		// PersistFields covers the static, non-connection inputs only.
+		// Connection-section fields (provider picker + per-provider
+		// inputs) are schema-driven now: their DOM ids and
+		// localStorage keys are derived at runtime from the host's
+		// /clone/connection/schemas response and persistence is hooked
+		// up by Pict-Provider-DataCloner#bootstrapConnectionSchemas
+		// after the schema-driven Connection view re-renders.  See
+		// PictView-DataCloner-Connection.js for the field-id
+		// convention.
 		this.pict.AppData.DataCloner =
 		{
 			FetchedTables: [],
@@ -67,30 +92,32 @@ class DataClonerApplication extends libPictApplication
 				'serverURL', 'authMethod', 'authURI', 'checkURI',
 				'cookieName', 'cookieValueAddr', 'cookieValueTemplate', 'loginMarker',
 				'userName', 'password', 'schemaURL', 'pageSize', 'dateTimePrecisionMS',
-				'connProvider', 'sqliteFilePath',
-				'mysqlServer', 'mysqlPort', 'mysqlUser', 'mysqlPassword', 'mysqlDatabase', 'mysqlConnectionLimit',
-				'mssqlServer', 'mssqlPort', 'mssqlUser', 'mssqlPassword', 'mssqlDatabase', 'mssqlConnectionLimit',
-				'mssqlRequestTimeoutSec', 'mssqlConnectionTimeoutSec',
-				'mssqlConnectMaxAttempts', 'mssqlDDLMaxAttempts',
-				'mssqlRetryInitialDelaySec', 'mssqlRetryMaxDelaySec',
-				'postgresqlHost', 'postgresqlPort', 'postgresqlUser', 'postgresqlPassword', 'postgresqlDatabase', 'postgresqlConnectionLimit',
-				'solrHost', 'solrPort', 'solrCore', 'solrPath',
-				'mongodbHost', 'mongodbPort', 'mongodbUser', 'mongodbPassword', 'mongodbDatabase', 'mongodbConnectionLimit',
-				'rocksdbFolder',
-				'bibliographFolder',
 				'syncMaxRecords'
-			]
+			],
+			// Connection state — populated by bootstrapConnectionSchemas().
+			// Initialized empty here so the Connection view's first
+			// onBeforeRender finds a valid (if empty) shape.
+			Connection:
+			{
+				Schemas:         [],
+				ActiveProvider:  '',
+				ProviderOptions: [],
+				ProviderForms:   [],
+				NoSchemasSlot:   [{}],
+				PreviewText:     'Loading providers…'
+			}
 		};
 
 		// Make pict available for inline onclick handlers
 		window.pict = this.pict;
 
-		// Render layout (which chains child view renders via onAfterRender)
+		// Render layout (which chains child view renders via onAfterRender).
+		// The Connection view renders an empty shell here — the schemas
+		// arrive asynchronously and trigger a re-render once they land.
 		this.pict.views['DataCloner-Layout'].render();
 
-		// Post-render initialization
+		// Post-render initialization for the static (non-connection) UI.
 		this.pict.providers.DataCloner.initPersistence();
-		this.pict.views['DataCloner-Connection'].onProviderChange();
 		this.pict.providers.DataCloner.restoreDeployedTables();
 		this.pict.providers.DataCloner.startLiveStatusPolling();
 		this.pict.providers.DataCloner.initAccordionPreviews();
@@ -98,6 +125,13 @@ class DataClonerApplication extends libPictApplication
 		this.pict.views['DataCloner-Layout'].collapseAllSections();
 		this.pict.providers.DataCloner.initAutoProcess();
 
+		// Async: fetch the host's connection-form schemas and re-render
+		// the Connection section.  bootstrapConnectionSchemas restores
+		// localStorage values + hooks save listeners once the new DOM
+		// is in place, then invokes onProviderChange() to surface the
+		// active provider's form.
+		this.pict.providers.DataCloner.bootstrapConnectionSchemas(function () { /* fire-and-forget */ });
+
 		return fCallback();
 	}
 }