retold-data-service 2.0.18 → 2.0.19

package/test/integration-report.json

@@ -1,11 +1,11 @@
 {
-	"timestamp": "2026-03-08T20:33:17.745Z",
-	"duration_ms": 54218,
+	"timestamp": "2026-03-09T04:55:24.920Z",
+	"duration_ms": 22241,
 	"summary": {
-		"total": 35,
-		"passed": 32,
+		"total": 26,
+		"passed": 26,
 		"failed": 0,
-		"skipped": 3
+		"skipped": 0
 	},
 	"storage_engines": {},
 	"suites": [
@@ -20,7 +20,7 @@
 				{
 					"name": "Should show initial connection status",
 					"status": "pass",
-					"duration_ms": 1
+					"duration_ms": 2
 				},
 				{
 					"name": "Should connect SQLite via configure",
@@ -30,10 +30,10 @@
 				{
 					"name": "Should show connected after configure",
 					"status": "pass",
-					"duration_ms": 0
+					"duration_ms": 1
 				}
 			],
-			"duration_ms": 3
+			"duration_ms": 5
 		},
 		{
 			"name": "Session Configuration",
@@ -46,7 +46,7 @@
 				{
 					"name": "Should show session as configured",
 					"status": "pass",
-					"duration_ms": 9
+					"duration_ms": 10
 				},
 				{
 					"name": "Should reject session configure without ServerURL",
@@ -54,7 +54,7 @@
 					"duration_ms": 1
 				}
 			],
-			"duration_ms": 11
+			"duration_ms": 12
 		},
 		{
 			"name": "Schema Fetch",
@@ -89,7 +89,7 @@
 				{
 					"name": "Should start initial sync with record cap",
 					"status": "pass",
-					"duration_ms": 1007
+					"duration_ms": 1005
 				},
 				{
 					"name": "All tables should have completed",
@@ -99,7 +99,7 @@
 				{
 					"name": "Should have a valid sync report",
 					"status": "pass",
-					"duration_ms": 3
+					"duration_ms": 2
 				},
 				{
 					"name": "Report tables should have timing data",
@@ -107,7 +107,7 @@
 					"duration_ms": 1
 				}
 			],
-			"duration_ms": 1012
+			"duration_ms": 1009
 		},
 		{
 			"name": "Data Integrity",
@@ -115,12 +115,12 @@
 				{
 					"name": "Should fetch harness reference data",
 					"status": "pass",
-					"duration_ms": 9
+					"duration_ms": 7
 				},
 				{
 					"name": "Local book count should match sync (capped)",
 					"status": "pass",
-					"duration_ms": 2
+					"duration_ms": 1
 				},
 				{
 					"name": "Local Book 1 should match harness data",
@@ -133,7 +133,7 @@
 					"duration_ms": 0
 				}
 			],
-			"duration_ms": 12
+			"duration_ms": 9
 		},
 		{
 			"name": "Pre-Count and Live Status",
@@ -141,10 +141,10 @@
 				{
 					"name": "Should run a fresh pipeline and capture pre-count data",
 					"status": "pass",
-					"duration_ms": 7885
+					"duration_ms": 8863
 				}
 			],
-			"duration_ms": 7885
+			"duration_ms": 8863
 		},
 		{
 			"name": "Ongoing Sync",
@@ -152,7 +152,7 @@
 				{
 					"name": "Should run ongoing sync after initial",
 					"status": "pass",
-					"duration_ms": 1019
+					"duration_ms": 1015
 				},
 				{
 					"name": "Ongoing sync report should show success",
@@ -160,7 +160,7 @@
 					"duration_ms": 1
 				}
 			],
-			"duration_ms": 1020
+			"duration_ms": 1016
 		},
 		{
 			"name": "Stop Sync",
@@ -168,10 +168,10 @@
 				{
 					"name": "Should be able to stop a sync in progress",
 					"status": "pass",
-					"duration_ms": 4063
+					"duration_ms": 4020
 				}
 			],
-			"duration_ms": 4063
+			"duration_ms": 4020
 		},
 		{
 			"name": "Reset",
@@ -179,7 +179,7 @@
 				{
 					"name": "Should reset the database",
 					"status": "pass",
-					"duration_ms": 2
+					"duration_ms": 1
 				},
 				{
 					"name": "Connection should still work after reset",
@@ -187,125 +187,44 @@
 					"duration_ms": 1
 				}
 			],
-			"duration_ms": 3
+			"duration_ms": 2
 		},
 		{
 			"name": "Storage Engine: MySQL",
 			"tests": [
 				{
 					"name": "Should sync via MySQL",
-					"status": "skip",
-					"duration_ms": 0
+					"status": "pass",
+					"duration_ms": 1033
 				}
 			],
-			"duration_ms": 0
+			"duration_ms": 1033
 		},
 		{
 			"name": "Storage Engine: PostgreSQL",
 			"tests": [
 				{
 					"name": "Should sync via PostgreSQL",
-					"status": "skip",
-					"duration_ms": 0
+					"status": "pass",
+					"duration_ms": 1035
 				}
 			],
-			"duration_ms": 0
+			"duration_ms": 1035
 		},
 		{
 			"name": "Storage Engine: MSSQL",
 			"tests": [
 				{
 					"name": "Should sync via MSSQL",
-					"status": "skip",
-					"duration_ms": 0
+					"status": "pass",
+					"duration_ms": 3103
 				}
 			],
-			"duration_ms": 0
+			"duration_ms": 3103
 		}
 	],
 	"puppeteer": {
-		"available": true,
-		"suites": [
-			{
-				"name": "Data Cloner Web UI (Puppeteer)",
-				"tests": [],
-				"duration_ms": 0
-			},
-			{
-				"name": "Web UI Load",
-				"tests": [
-					{
-						"name": "Should load the data cloner page",
-						"status": "pass",
-						"duration_ms": 734
-					},
-					{
-						"name": "Should display 7 accordion sections",
-						"status": "pass",
-						"duration_ms": 15
-					},
-					{
-						"name": "Should show live status bar",
-						"status": "pass",
-						"duration_ms": 201
-					}
-				],
-				"duration_ms": 950
-			},
-			{
-				"name": "Connection Section",
-				"tests": [
-					{
-						"name": "Should connect to SQLite via go button",
-						"status": "pass",
-						"duration_ms": 2218
-					}
-				],
-				"duration_ms": 2218
-			},
-			{
-				"name": "Session, Schema, and Deploy Flow",
-				"tests": [
-					{
-						"name": "Should configure session via UI",
-						"status": "pass",
-						"duration_ms": 3019
-					},
-					{
-						"name": "Should fetch schema via UI",
-						"status": "pass",
-						"duration_ms": 2020
-					},
-					{
-						"name": "Should deploy schema via UI",
-						"status": "pass",
-						"duration_ms": 5074
-					}
-				],
-				"duration_ms": 10113
-			},
-			{
-				"name": "Sync via Web UI",
-				"tests": [
-					{
-						"name": "Should start sync and complete successfully",
-						"status": "pass",
-						"duration_ms": 7184
-					}
-				],
-				"duration_ms": 7184
-			},
-			{
-				"name": "Live Status Display",
-				"tests": [
-					{
-						"name": "Should show progress information during sync",
-						"status": "pass",
-						"duration_ms": 16243
-					}
-				],
-				"duration_ms": 16243
-			}
-		]
+		"available": false,
+		"suites": []
 	}
 }

package/test/run-integration-tests.js

@@ -166,24 +166,54 @@ function fPrintReport()
 	console.log(`  Skipped:    ${_Report.summary.skipped}`);
 	console.log('-'.repeat(72));
 
-	// Storage engines
-	console.log('\n  Storage Engines:');
-	let tmpEngines = Object.keys(_Report.storage_engines);
-	for (let i = 0; i < tmpEngines.length; i++)
+	// Build storage engine status from suite results
+	let tmpStorageEnginePrefix = 'Storage Engine: ';
+	for (let i = 0; i < _Report.suites.length; i++)
 	{
-		let tmpName = tmpEngines[i];
-		let tmpEngine = _Report.storage_engines[tmpName];
-		if (tmpEngine.status === 'pass')
+		let tmpSuite = _Report.suites[i];
+		if (tmpSuite.name.indexOf(tmpStorageEnginePrefix) !== 0) continue;
+
+		let tmpEngineName = tmpSuite.name.substring(tmpStorageEnginePrefix.length);
+		let tmpPassCount = tmpSuite.tests.filter((t) => t.status === 'pass').length;
+		let tmpFailCount = tmpSuite.tests.filter((t) => t.status === 'fail').length;
+		let tmpSkipCount = tmpSuite.tests.filter((t) => t.status === 'skip').length;
+
+		if (tmpSkipCount === tmpSuite.tests.length)
 		{
-			console.log(`    ${tmpName.padEnd(14)} PASS   ${fFormatDuration(tmpEngine.sync_duration_ms).padStart(10)}   ${tmpEngine.records_synced} records / ${tmpEngine.tables_synced} tables`);
+			_Report.storage_engines[tmpEngineName] = { status: 'skip', reason: 'Not configured (env vars not set)' };
 		}
-		else if (tmpEngine.status === 'fail')
+		else if (tmpFailCount > 0)
 		{
-			console.log(`    ${tmpName.padEnd(14)} FAIL   ${tmpEngine.error || 'Unknown error'}`);
+			let tmpFirstError = tmpSuite.tests.find((t) => t.status === 'fail');
+			_Report.storage_engines[tmpEngineName] = { status: 'fail', error: tmpFirstError ? tmpFirstError.error : 'Unknown' };
 		}
 		else
 		{
-			console.log(`    ${tmpName.padEnd(14)} SKIP   ${tmpEngine.reason || ''}`);
+			_Report.storage_engines[tmpEngineName] = { status: 'pass', sync_duration_ms: tmpSuite.duration_ms, records_synced: 0, tables_synced: 0 };
+		}
+	}
+
+	// Storage engines
+	let tmpEngines = Object.keys(_Report.storage_engines);
+	if (tmpEngines.length > 0)
+	{
+		console.log('\n  Storage Engines:');
+		for (let i = 0; i < tmpEngines.length; i++)
+		{
+			let tmpName = tmpEngines[i];
+			let tmpEngine = _Report.storage_engines[tmpName];
+			if (tmpEngine.status === 'pass')
+			{
+				console.log(`    ${tmpName.padEnd(14)} PASS   ${fFormatDuration(tmpEngine.sync_duration_ms).padStart(10)}   ${tmpEngine.records_synced} records / ${tmpEngine.tables_synced} tables`);
+			}
+			else if (tmpEngine.status === 'fail')
+			{
+				console.log(`    ${tmpName.padEnd(14)} FAIL   ${tmpEngine.error || 'Unknown error'}`);
+			}
+			else
+			{
+				console.log(`    ${tmpName.padEnd(14)} SKIP   ${tmpEngine.reason || ''}`);
+			}
 		}
 	}
 
@@ -194,8 +224,18 @@ function fPrintReport()
 		let tmpSuite = _Report.suites[i];
 		let tmpPassCount = tmpSuite.tests.filter((t) => t.status === 'pass').length;
 		let tmpFailCount = tmpSuite.tests.filter((t) => t.status === 'fail').length;
+		let tmpSkipCount = tmpSuite.tests.filter((t) => t.status === 'skip').length;
+
+		if (tmpSkipCount === tmpSuite.tests.length && tmpSuite.tests.length > 0)
+		{
+			// All tests in this suite were skipped
+			console.log(`    ~ ${tmpSuite.name.padEnd(35)} skipped`);
+			continue;
+		}
+
 		let tmpStatus = tmpFailCount > 0 ? 'FAIL' : 'PASS';
-		console.log(`    ${tmpStatus === 'FAIL' ? 'X' : '+'} ${tmpSuite.name.padEnd(35)} ${tmpPassCount}/${tmpSuite.tests.length} passed   ${fFormatDuration(tmpSuite.duration_ms)}`);
+		let tmpSkipLabel = tmpSkipCount > 0 ? ` (${tmpSkipCount} skipped)` : '';
+		console.log(`    ${tmpStatus === 'FAIL' ? 'X' : '+'} ${tmpSuite.name.padEnd(35)} ${tmpPassCount}/${tmpPassCount + tmpFailCount} passed   ${fFormatDuration(tmpSuite.duration_ms)}${tmpSkipLabel}`);
 
 		// Show failed tests
 		for (let j = 0; j < tmpSuite.tests.length; j++)

package/test/run-integration-tests.sh

@@ -0,0 +1,221 @@
+#!/bin/bash
+# ==============================================================================
+# Data Cloner Integration Tests — All Storage Engines
+#
+# Starts Docker containers for MySQL, PostgreSQL, and MSSQL, creates the
+# required test databases, then runs the full integration test suite against
+# every engine (SQLite runs without Docker).
+#
+# Usage:
+#   ./test/run-integration-tests.sh              # All engines
+#   ./test/run-integration-tests.sh sqlite        # SQLite only (no Docker)
+#   ./test/run-integration-tests.sh mysql         # MySQL only
+#   ./test/run-integration-tests.sh mysql,mssql   # Specific engines
+#
+# Prerequisites:
+#   - Docker (and docker compose) installed and running
+#   - Node.js and npm
+#   - npm install already run in retold-data-service
+# ==============================================================================
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+HARNESS_DIR="$(cd "$PROJECT_DIR/../retold-harness" 2>/dev/null && pwd)" || true
+
+# ---- Parse arguments ----
+ENGINES="${1:-sqlite,mysql,postgresql,mssql}"
+
+# ---- Credentials (match retold-harness/docker-compose.yml) ----
+MYSQL_ROOT_PASSWORD="1234567890"
+MYSQL_TEST_DB="retold_cloner_test"
+
+POSTGRESQL_PASSWORD="retold1234567890"
+POSTGRESQL_TEST_DB="retold_cloner_test"
+
+MSSQL_SA_PASSWORD="Retold1234567890!"
+MSSQL_TEST_DB="retold_cloner_test"
+
+# ---- Colours ----
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+CYAN='\033[0;36m'
+NC='\033[0m' # No colour
+
+log()  { echo -e "${CYAN}[$(date +%H:%M:%S)]${NC} $*"; }
+warn() { echo -e "${YELLOW}[$(date +%H:%M:%S)] WARNING:${NC} $*"; }
+err()  { echo -e "${RED}[$(date +%H:%M:%S)] ERROR:${NC} $*"; }
+ok()   { echo -e "${GREEN}[$(date +%H:%M:%S)] OK:${NC} $*"; }
+
+needs_docker()
+{
+	[[ "$ENGINES" == *mysql* ]] || [[ "$ENGINES" == *postgresql* ]] || [[ "$ENGINES" == *mssql* ]]
+}
+
+# ---- Check prerequisites ----
+if needs_docker; then
+	if ! command -v docker &>/dev/null; then
+		err "Docker is not installed. Install Docker to test non-SQLite engines."
+		err "Or run:  $0 sqlite"
+		exit 1
+	fi
+	if ! docker info &>/dev/null; then
+		err "Docker daemon is not running. Start Docker first."
+		exit 1
+	fi
+fi
+
+if [ -z "$HARNESS_DIR" ] || [ ! -d "$HARNESS_DIR" ]; then
+	err "retold-harness not found at $PROJECT_DIR/../retold-harness"
+	err "Make sure retold-harness is cloned as a sibling of retold-data-service."
+	exit 1
+fi
+
+# ---- Start Docker containers ----
+start_containers()
+{
+	log "Starting Docker containers..."
+	cd "$HARNESS_DIR"
+
+	local SERVICES=""
+	[[ "$ENGINES" == *mysql* ]]      && SERVICES="$SERVICES mysql"
+	[[ "$ENGINES" == *postgresql* ]] && SERVICES="$SERVICES postgresql"
+	[[ "$ENGINES" == *mssql* ]]      && SERVICES="$SERVICES mssql"
+
+	if [ -n "$SERVICES" ]; then
+		docker compose up -d $SERVICES
+		log "Waiting for containers to become healthy..."
+	fi
+
+	cd "$PROJECT_DIR"
+}
+
+# Wait for a container to be healthy (or timeout after N seconds)
+wait_healthy()
+{
+	local CONTAINER="$1"
+	local TIMEOUT="${2:-120}"
+	local ELAPSED=0
+
+	while [ $ELAPSED -lt $TIMEOUT ]; do
+		local STATUS
+		STATUS="$(docker inspect --format='{{.State.Health.Status}}' "$CONTAINER" 2>/dev/null || echo "missing")"
+		if [ "$STATUS" = "healthy" ]; then
+			return 0
+		fi
+		sleep 2
+		ELAPSED=$((ELAPSED + 2))
+	done
+
+	err "$CONTAINER did not become healthy within ${TIMEOUT}s (status: $STATUS)"
+	return 1
+}
+
+# ---- Create test databases ----
+create_mysql_db()
+{
+	log "Waiting for MySQL to be healthy..."
+	wait_healthy "retold-harness-mysql" 120
+
+	log "Creating MySQL database '${MYSQL_TEST_DB}'..."
+	docker exec retold-harness-mysql mysql \
+		-uroot "-p${MYSQL_ROOT_PASSWORD}" \
+		-e "CREATE DATABASE IF NOT EXISTS \`${MYSQL_TEST_DB}\`;" 2>/dev/null
+	ok "MySQL database '${MYSQL_TEST_DB}' ready"
+}
+
+create_postgresql_db()
+{
+	log "Waiting for PostgreSQL to be healthy..."
+	wait_healthy "retold-harness-postgresql" 120
+
+	log "Creating PostgreSQL database '${POSTGRESQL_TEST_DB}'..."
+	# createdb returns 0 even if DB exists when using IF NOT EXISTS via psql
+	docker exec retold-harness-postgresql psql \
+		-U postgres \
+		-tc "SELECT 1 FROM pg_database WHERE datname = '${POSTGRESQL_TEST_DB}'" \
+		| grep -q 1 \
+		|| docker exec retold-harness-postgresql psql \
+			-U postgres \
+			-c "CREATE DATABASE ${POSTGRESQL_TEST_DB};"
+	ok "PostgreSQL database '${POSTGRESQL_TEST_DB}' ready"
+}
+
+create_mssql_db()
+{
+	log "Waiting for MSSQL to be healthy..."
+	wait_healthy "retold-harness-mssql" 120
+
+	log "Creating MSSQL database '${MSSQL_TEST_DB}'..."
+	docker exec retold-harness-mssql /opt/mssql-tools18/bin/sqlcmd \
+		-S localhost -U sa -P "${MSSQL_SA_PASSWORD}" -C \
+		-Q "IF NOT EXISTS (SELECT name FROM sys.databases WHERE name = '${MSSQL_TEST_DB}') CREATE DATABASE [${MSSQL_TEST_DB}];" 2>/dev/null
+	ok "MSSQL database '${MSSQL_TEST_DB}' ready"
+}
+
+# ---- Main ----
+
+log "Engines requested: ${ENGINES}"
+log "Project dir: ${PROJECT_DIR}"
+log "Harness dir: ${HARNESS_DIR}"
+echo ""
+
+# Start Docker containers if needed
+if needs_docker; then
+	start_containers
+
+	# Create test databases in parallel where possible
+	[[ "$ENGINES" == *mysql* ]]      && create_mysql_db
+	[[ "$ENGINES" == *postgresql* ]] && create_postgresql_db
+	[[ "$ENGINES" == *mssql* ]]      && create_mssql_db
+
+	echo ""
+fi
+
+# ---- Set environment variables ----
+if [[ "$ENGINES" == *mysql* ]]; then
+	export MYSQL_HOST="localhost"
+	export MYSQL_PORT="3306"
+	export MYSQL_USER="root"
+	export MYSQL_PASSWORD="${MYSQL_ROOT_PASSWORD}"
+	export MYSQL_DATABASE="${MYSQL_TEST_DB}"
+	ok "MySQL env configured (localhost:3306, db=${MYSQL_TEST_DB})"
+fi
+
+if [[ "$ENGINES" == *postgresql* ]]; then
+	export POSTGRESQL_HOST="localhost"
+	export POSTGRESQL_PORT="5432"
+	export POSTGRESQL_USER="postgres"
+	export POSTGRESQL_PASSWORD="${POSTGRESQL_PASSWORD}"
+	export POSTGRESQL_DATABASE="${POSTGRESQL_TEST_DB}"
+	ok "PostgreSQL env configured (localhost:5432, db=${POSTGRESQL_TEST_DB})"
+fi
+
+if [[ "$ENGINES" == *mssql* ]]; then
+	export MSSQL_HOST="localhost"
+	export MSSQL_PORT="1433"
+	export MSSQL_USER="sa"
+	export MSSQL_PASSWORD="${MSSQL_SA_PASSWORD}"
+	export MSSQL_DATABASE="${MSSQL_TEST_DB}"
+	ok "MSSQL env configured (localhost:1433, db=${MSSQL_TEST_DB})"
+fi
+
+echo ""
+log "Running integration tests..."
+echo ""
+
+# ---- Run tests ----
+cd "$PROJECT_DIR"
+node test/run-integration-tests.js --skip-puppeteer --engines="${ENGINES}"
+TEST_EXIT=$?
+
+echo ""
+if [ $TEST_EXIT -eq 0 ]; then
+	ok "All integration tests passed!"
+else
+	err "Some tests failed (exit code: ${TEST_EXIT})"
+fi
+
+exit $TEST_EXIT