xlsform2lstsv 0.2.0

package/dist/generateFixtures.js

@@ -0,0 +1,123 @@
+#!/usr/bin/env node
+/**
+ * Generate TSV files from XLSForm fixtures for integration testing.
+ *
+ * This script:
+ * 1. Reads XLSForm JSON and XLSX fixtures from docker_tests/fixtures/
+ * 2. Converts them to LimeSurvey TSV format
+ * 3. Saves output to docker_tests/integration/output/
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+import { XLSLoader } from './processors/XLSLoader.js';
+import { XLSFormToTSVConverter } from './xlsformConverter.js';
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const FIXTURES_DIR = path.join(__dirname, '../docker_tests/fixtures');
+const OUTPUT_DIR = path.join(__dirname, '../docker_tests/integration/output');
+function ensureDirectoryExists(dir) {
+    if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+    }
+}
+function cleanOutputDirectory(dir) {
+    if (fs.existsSync(dir)) {
+        // Remove all .tsv files in the output directory
+        const files = fs.readdirSync(dir);
+        for (const file of files) {
+            if (file.endsWith('.tsv')) {
+                fs.unlinkSync(path.join(dir, file));
+            }
+        }
+    }
+}
+async function generateTSVFromFixture(fixturePath, outputPath) {
+    console.log(`Processing: ${path.basename(fixturePath)}`);
+    // Read fixture
+    const fixtureContent = fs.readFileSync(fixturePath, 'utf-8');
+    const fixture = JSON.parse(fixtureContent);
+    // Convert to TSV with configuration that removes underscores but doesn't truncate field names
+    // This matches LimeSurvey's behavior (removes underscores but allows longer field names)
+    // Answer codes are already limited to 5 chars in the fixtures
+    const converter = new XLSFormToTSVConverter({});
+    const tsv = await converter.convert(fixture.survey, fixture.choices, fixture.settings);
+    // Write output
+    fs.writeFileSync(outputPath, tsv, 'utf-8');
+    console.log(`  → Generated: ${path.basename(outputPath)}`);
+    // Print stats
+    const lines = tsv.split('\n').filter(l => l.trim()).length;
+    console.log(`  → ${lines} rows (including header)`);
+}
+async function generateTSVFromXLSX(xlsxPath, outputPath) {
+    console.log(`Processing: ${path.basename(xlsxPath)}`);
+    // Read and parse xlsx file
+    const fileData = fs.readFileSync(xlsxPath);
+    const { surveyData, choicesData, settingsData } = XLSLoader.parseXLSData(fileData, { skipValidation: true });
+    // Convert to TSV
+    const converter = new XLSFormToTSVConverter({});
+    const tsv = await converter.convert(surveyData, choicesData, settingsData);
+    // Write output
+    fs.writeFileSync(outputPath, tsv, 'utf-8');
+    console.log(`  → Generated: ${path.basename(outputPath)}`);
+    // Print stats
+    const lines = tsv.split('\n').filter(l => l.trim()).length;
+    console.log(`  → ${lines} rows (including header)`);
+}
+async function main() {
+    console.log('Generating TSV files from XLSForm fixtures...\n');
+    // Ensure output directory exists and clean old files
+    ensureDirectoryExists(OUTPUT_DIR);
+    cleanOutputDirectory(OUTPUT_DIR);
+    console.log('Cleaned output directory\n');
+    // Find all JSON fixtures
+    const jsonFiles = fs.readdirSync(FIXTURES_DIR)
+        .filter(file => file.endsWith('.json'))
+        .map(file => path.join(FIXTURES_DIR, file));
+    // Find all XLSX fixtures
+    const xlsxFiles = fs.readdirSync(FIXTURES_DIR)
+        .filter(file => file.endsWith('.xlsx'))
+        .map(file => path.join(FIXTURES_DIR, file));
+    const totalFiles = jsonFiles.length + xlsxFiles.length;
+    if (totalFiles === 0) {
+        console.error('No fixture files found in', FIXTURES_DIR);
+        process.exit(1);
+    }
+    // Generate TSV for each JSON fixture
+    let jsonSuccessCount = 0;
+    for (const fixturePath of jsonFiles) {
+        const baseName = path.basename(fixturePath, '.json');
+        const outputPath = path.join(OUTPUT_DIR, `${baseName}.tsv`);
+        try {
+            await generateTSVFromFixture(fixturePath, outputPath);
+            jsonSuccessCount++;
+        }
+        catch (error) {
+            console.error(`  ✗ Error processing ${baseName}:`, error);
+        }
+        console.log('');
+    }
+    // Generate TSV for each XLSX fixture
+    // XLSX files may contain unimplemented types (e.g. range) — failures are logged but not fatal
+    let xlsxSuccessCount = 0;
+    for (const xlsxPath of xlsxFiles) {
+        const baseName = path.basename(xlsxPath, '.xlsx');
+        const outputPath = path.join(OUTPUT_DIR, `${baseName}.tsv`);
+        try {
+            await generateTSVFromXLSX(xlsxPath, outputPath);
+            xlsxSuccessCount++;
+        }
+        catch (error) {
+            console.warn(`  ⚠ Skipped ${baseName}:`, error.message);
+        }
+        console.log('');
+    }
+    const totalSuccess = jsonSuccessCount + xlsxSuccessCount;
+    console.log(`\nFinal Summary: ${totalSuccess}/${totalFiles} files generated successfully`);
+    // Only fail if JSON fixtures (which should always succeed) had errors
+    if (jsonSuccessCount < jsonFiles.length) {
+        process.exit(1);
+    }
+}
+void (async () => {
+    await main();
+})();

package/dist/index.js

@@ -0,0 +1,10 @@
+export { XLSFormToTSVConverter } from './xlsformConverter.js';
+export { convertRelevance, convertConstraint, xpathToLimeSurvey } from './converters/xpathTranspiler.js';
+export { ConfigManager } from './config/ConfigManager.js';
+export { defaultConfig } from './config/types.js';
+export { FieldSanitizer } from './processors/FieldSanitizer.js';
+export { TypeMapper, TYPE_MAPPINGS } from './processors/TypeMapper.js';
+export { TSVGenerator } from './processors/TSVGenerator.js';
+export { XLSFormParser } from './processors/XLSFormParser.js';
+export { XLSLoader } from './processors/XLSLoader.js';
+export { XLSValidator } from './processors/XLSValidator.js';

package/dist/processors/FieldSanitizer.js

@@ -0,0 +1,21 @@
+import { sanitizeFieldName } from '../utils/helpers.js';
+export class FieldSanitizer {
+    constructor() { }
+    sanitizeName(name) {
+        return sanitizeFieldName(name);
+    }
+    sanitizeAnswerCode(code) {
+        // Answer codes in LimeSurvey have a 5-character limit
+        let result = code;
+        // Always remove underscores and hyphens (LimeSurvey only allows alphanumeric)
+        result = result.replace(/[_-]/g, '');
+        // Limit to 5 characters (LimeSurvey answer codes limit)
+        const maxLength = 5;
+        if (result.length > maxLength) {
+            const truncated = result.substring(0, maxLength);
+            console.warn(`Answer code "${code}" exceeds maximum length of ${maxLength} characters and will be truncated to "${truncated}"`);
+            return truncated;
+        }
+        return result;
+    }
+}

package/dist/processors/TSVGenerator.js

@@ -0,0 +1,52 @@
+export class TSVGenerator {
+    constructor() {
+        this.rows = [];
+    }
+    addRow(row) {
+        this.rows.push(row);
+    }
+    // https://www.limesurvey.org/manual/Tab_Separated_Value_survey_structure
+    generateTSV() {
+        const headers = [
+            'class',
+            'type/scale',
+            'name',
+            'relevance',
+            'text',
+            'help',
+            'language',
+            'validation',
+            'em_validation_q',
+            'mandatory',
+            'other',
+            'default',
+            'same_default'
+        ];
+        const lines = [headers.join('\t')];
+        for (const row of this.rows) {
+            const values = headers.map((h) => this.escapeForTSV(row[h] || ''));
+            lines.push(values.join('\t'));
+        }
+        return lines.join('\n');
+    }
+    escapeForTSV(value) {
+        // Escape tabs, newlines, and wrap in quotes if needed
+        if (typeof value !== 'string')
+            value = String(value);
+        // Replace newlines with <br /> for LimeSurvey compatibility.
+        // LimeSurvey's TSV importer parses line-by-line and does not handle
+        // RFC 4180 multi-line quoted fields. HTML breaks render correctly
+        // in LimeSurvey's question text and help fields.
+        value = value.replace(/\r\n/g, '<br />').replace(/\n/g, '<br />');
+        if (value.includes('\t') || value.includes('"')) {
+            return '"' + value.replace(/"/g, '""') + '"';
+        }
+        return value;
+    }
+    clear() {
+        this.rows = [];
+    }
+    getRowCount() {
+        return this.rows.length;
+    }
+}

package/dist/processors/TypeMapper.js

@@ -0,0 +1,55 @@
+/**
+ * @file Mapping between XLSForm question types and LimeSurvey question types.
+ * @description See https://xlsform.org/en/ref-table/#types and
+ * https://www.limesurvey.org/manual/Tab_Separated_Value_survey_structure
+ */
+export const TYPE_MAPPINGS = {
+    // Text types
+    text: { limeSurveyType: 'S' },
+    string: { limeSurveyType: 'S' },
+    // Numeric types
+    integer: { limeSurveyType: 'N' },
+    int: { limeSurveyType: 'N' },
+    decimal: { limeSurveyType: 'N' },
+    // Date/time
+    date: { limeSurveyType: 'D' },
+    time: { limeSurveyType: 'D' },
+    datetime: { limeSurveyType: 'D' },
+    // Select types
+    select_one: { limeSurveyType: 'L', supportsOther: true, answerClass: 'A' },
+    select_multiple: { limeSurveyType: 'M', supportsOther: true, answerClass: 'SQ' },
+    // Other types
+    note: { limeSurveyType: 'X' },
+    rank: { limeSurveyType: 'R', answerClass: 'A', supportsOther: true },
+};
+export class TypeMapper {
+    parseType(typeStr) {
+        const parts = typeStr.split(/\s+/);
+        const base = parts[0];
+        let listName = null;
+        let orOther = false;
+        if (base === 'select_one' || base === 'select_multiple' || base === 'rank') {
+            listName = parts[1] || null;
+            orOther = parts.includes('or_other');
+        }
+        return { base, listName, orOther };
+    }
+    mapType(typeInfo) {
+        const mapping = TYPE_MAPPINGS[typeInfo.base];
+        if (!mapping) {
+            // Fallback to text type for unknown types
+            console.warn(`No type mapping found for "${typeInfo.base}", defaulting to text type`);
+            return { type: 'S' };
+        }
+        const result = {
+            type: mapping.limeSurveyType,
+        };
+        if (mapping.supportsOther && typeInfo.orOther) {
+            result.other = true;
+        }
+        if (mapping.answerClass) {
+            result.answerClass = mapping.answerClass;
+        }
+        return result;
+    }
+}

package/dist/processors/XLSFormParser.js

@@ -0,0 +1,32 @@
+/**
+ * @file Main entrypoint of this library.
+ */
+import { XLSLoader } from './XLSLoader.js';
+export class XLSFormParser {
+    /**
+     * Convert XLS/XLSX file to TSV using the main converter
+     * @param filePath Path to XLS or XLSX file
+     * @param config Optional configuration
+     * @returns TSV string
+     */
+    static async convertXLSFileToTSV(filePath, config) {
+        const { XLSFormToTSVConverter } = await import('../xlsformConverter.js');
+        // Load data (validation is included by default)
+        const { surveyData, choicesData, settingsData } = XLSLoader.parseXLSFile(filePath);
+        const converter = new XLSFormToTSVConverter(config);
+        return await converter.convert(surveyData, choicesData, settingsData);
+    }
+    /**
+     * Convert XLS/XLSX data to TSV using the main converter
+     * @param data XLS or XLSX file data
+     * @param config Optional configuration
+     * @returns TSV string
+     */
+    static async convertXLSDataToTSV(data, config) {
+        const { XLSFormToTSVConverter } = await import('../xlsformConverter.js');
+        // Load data (validation is included by default)
+        const { surveyData, choicesData, settingsData } = XLSLoader.parseXLSData(data);
+        const converter = new XLSFormToTSVConverter(config);
+        return await converter.convert(surveyData, choicesData, settingsData);
+    }
+}

package/dist/processors/XLSLoader.js

@@ -0,0 +1,109 @@
+import * as XLSX from 'xlsx';
+import { extractBaseColumnName, extractLanguageCode, getLanguageCodesFromHeaders, validateLanguageCodes, isValidLanguageCode } from '../utils/languageUtils.js';
+import { XLSValidator } from './XLSValidator.js';
+export class XLSLoader {
+    /**
+     * Parse XLS/XLSX file and extract survey data with validation
+     * @param filePath Path to XLS or XLSX file
+     * @param options Validation options
+     * @returns Object containing validated survey, choices, and settings data
+     * @throws Error if required sheets or columns are missing
+     */
+    static parseXLSFile(filePath, options = {}) {
+        const workbook = XLSX.readFile(filePath);
+        return this.parseWorkbook(workbook, options);
+    }
+    /**
+     * Parse XLS/XLSX data (Buffer or ArrayBuffer) with validation
+     * @param data XLS or XLSX file data
+     * @param options Validation options
+     * @returns Object containing validated survey, choices, and settings data
+     * @throws Error if required sheets or columns are missing
+     */
+    static parseXLSData(data, options = {}) {
+        const workbook = XLSX.read(data);
+        return this.parseWorkbook(workbook, options);
+    }
+    /**
+     * Parse XLSX workbook with validation
+     * @param workbook XLSX workbook object
+     * @param options Validation options
+     * @returns Object containing validated survey, choices, and settings data
+     * @throws Error if required sheets or columns are missing
+     */
+    static parseWorkbook(workbook, options = {}) {
+        const surveyData = [];
+        const choicesData = [];
+        const settingsData = [];
+        // Track which sheets we've found
+        let hasSurveySheet = false;
+        let hasChoicesSheet = false;
+        let hasSettingsSheet = false;
+        // Process each sheet in the workbook
+        workbook.SheetNames.forEach(sheetName => {
+            const sheet = workbook.Sheets[sheetName];
+            const jsonData = XLSX.utils.sheet_to_json(sheet, { header: 1 });
+            if (!jsonData || jsonData.length === 0) {
+                return;
+            }
+            // Convert to objects with proper headers
+            const headers = jsonData[0];
+            const rows = jsonData.slice(1);
+            // Detect language codes from headers
+            const languageCodes = getLanguageCodesFromHeaders(headers);
+            // Validate language codes
+            const invalidLanguageCodes = validateLanguageCodes(languageCodes);
+            if (invalidLanguageCodes.length > 0) {
+                console.warn(`Warning: Invalid language codes detected in sheet "${sheetName}": ${invalidLanguageCodes.join(', ')}. These will be ignored. Valid language codes should be 2-letter IANA subtags (e.g., 'en', 'es', 'fr').`);
+            }
+            const sheetData = rows.map(row => {
+                const obj = {};
+                headers.forEach((header, index) => {
+                    if (header && row[index] !== undefined) {
+                        const baseColumn = extractBaseColumnName(header);
+                        const langCode = extractLanguageCode(header);
+                        // Store language-specific values in a structured way
+                        if (langCode) {
+                            // Only store valid language codes
+                            if (isValidLanguageCode(langCode)) {
+                                if (!obj[baseColumn]) {
+                                    obj[baseColumn] = {};
+                                }
+                                obj[baseColumn][langCode] = row[index];
+                            }
+                        }
+                        else {
+                            // Regular column
+                            obj[header] = row[index];
+                        }
+                    }
+                });
+                // Store detected languages for this sheet (only valid ones)
+                const validLanguages = languageCodes.filter(isValidLanguageCode);
+                if (validLanguages.length > 0) {
+                    obj._languages = validLanguages;
+                }
+                return obj;
+            });
+            // Match standard XLSForm sheet names (exact match, case-insensitive)
+            const lowerCaseSheetName = sheetName.toLowerCase().trim();
+            if (lowerCaseSheetName === 'survey') {
+                hasSurveySheet = true;
+                sheetData.forEach(row => surveyData.push(row));
+            }
+            else if (lowerCaseSheetName === 'choices' || lowerCaseSheetName === 'choice') {
+                hasChoicesSheet = true;
+                sheetData.forEach(row => choicesData.push(row));
+            }
+            else if (lowerCaseSheetName === 'settings' || lowerCaseSheetName === 'setting') {
+                hasSettingsSheet = true;
+                sheetData.forEach(row => settingsData.push(row));
+            }
+        });
+        // Validate by default, unless explicitly skipped
+        if (!options.skipValidation) {
+            XLSValidator.validateAll(surveyData, choicesData, hasSurveySheet, hasChoicesSheet);
+        }
+        return { surveyData, choicesData, settingsData, hasSurveySheet, hasChoicesSheet, hasSettingsSheet };
+    }
+}

package/dist/processors/XLSValidator.js

@@ -0,0 +1,121 @@
+export class XLSValidator {
+    /**
+     * Validate that required sheets are present
+     * @param hasSurveySheet Whether survey sheet was found
+     * @param hasChoicesSheet Whether choices sheet was found
+     * @throws Error if required sheets are missing
+     */
+    static validateRequiredSheets(hasSurveySheet, hasChoicesSheet) {
+        const missingSheets = [];
+        if (!hasSurveySheet)
+            missingSheets.push('survey');
+        if (!hasChoicesSheet)
+            missingSheets.push('choices');
+        if (missingSheets.length > 0) {
+            throw new Error(`XLSX file is missing required sheets: ${missingSheets.join(', ')}. An XLSForm must contain survey and choices sheets.`);
+        }
+    }
+    /**
+     * Validate that survey sheet has required columns
+     * @param data Survey data
+     * @param sheetName Name of the survey sheet
+     * @throws Error if required columns are missing
+     */
+    static validateSurveySheetColumns(data, sheetName) {
+        if (data.length === 0) {
+            console.warn(`Warning: Survey sheet "${sheetName}" is empty.`);
+            return;
+        }
+        // Collect all column keys that appear across any row
+        const allColumns = new Set();
+        for (const row of data) {
+            if (row && typeof row === 'object') {
+                for (const key of Object.keys(row)) {
+                    allColumns.add(key);
+                }
+            }
+        }
+        if (allColumns.size === 0) {
+            console.warn(`Warning: Survey sheet "${sheetName}" has no valid data rows.`);
+            return;
+        }
+        // Check for required columns across all rows
+        const requiredColumns = ['type', 'name', 'label'];
+        const missingColumns = requiredColumns.filter(col => !allColumns.has(col));
+        if (missingColumns.length > 0) {
+            throw new Error(`Survey sheet "${sheetName}" is missing required columns: ${missingColumns.join(', ')}. A survey sheet must contain type, name, and label columns.`);
+        }
+        // Warn about unexpected columns
+        const expectedColumns = ['type', 'name', 'label', 'hint', 'required', 'relevant', 'constraint', 'constraint_message', 'calculation', 'default', 'appearance', 'parameters'];
+        const unexpectedColumns = [...allColumns].filter(col => !expectedColumns.includes(col) && !col.startsWith('_'));
+        if (unexpectedColumns.length > 0) {
+            console.warn(`Warning: Survey sheet "${sheetName}" contains unexpected columns: ${unexpectedColumns.join(', ')}. These columns will be ignored.`);
+        }
+    }
+    /**
+     * Validate that choices sheet has required columns
+     * @param data Choices data
+     * @param sheetName Name of the choices sheet
+     * @throws Error if required columns are missing
+     */
+    static validateChoicesSheetColumns(data, sheetName) {
+        if (data.length === 0) {
+            console.warn(`Warning: Choices sheet "${sheetName}" is empty.`);
+            return;
+        }
+        // Collect all column keys that appear across any row
+        const allColumns = new Set();
+        for (const row of data) {
+            if (row && typeof row === 'object') {
+                for (const key of Object.keys(row)) {
+                    allColumns.add(key);
+                }
+            }
+        }
+        if (allColumns.size === 0) {
+            console.warn(`Warning: Choices sheet "${sheetName}" has no valid data rows.`);
+            return;
+        }
+        // Check for required columns (handle both list_name and list name variations)
+        const hasListName = allColumns.has('list_name') || allColumns.has('list name');
+        const hasName = allColumns.has('name');
+        const hasLabel = allColumns.has('label');
+        const missingColumns = [];
+        if (!hasListName)
+            missingColumns.push('list_name');
+        if (!hasName)
+            missingColumns.push('name');
+        if (!hasLabel)
+            missingColumns.push('label');
+        if (missingColumns.length > 0) {
+            throw new Error(`Choices sheet "${sheetName}" is missing required columns: ${missingColumns.join(', ')}. A choices sheet must contain list_name, name, and label columns.`);
+        }
+        // Warn about unexpected columns
+        const expectedColumns = ['list_name', 'list name', 'name', 'label', 'filter'];
+        const unexpectedColumns = [...allColumns].filter(col => !expectedColumns.includes(col) && !col.startsWith('_'));
+        if (unexpectedColumns.length > 0) {
+            console.warn(`Warning: Choices sheet "${sheetName}" contains unexpected columns: ${unexpectedColumns.join(', ')}. These columns will be ignored.`);
+        }
+    }
+    /**
+     * Validate all sheets in the parsed data
+     * @param surveyData Survey data
+     * @param choicesData Choices data
+     * @param hasSurveySheet Whether survey sheet was found
+     * @param hasChoicesSheet Whether choices sheet was found
+     * @param surveySheetName Name of the survey sheet
+     * @param choicesSheetName Name of the choices sheet
+     */
+    static validateAll(surveyData, choicesData, hasSurveySheet, hasChoicesSheet, surveySheetName = 'survey', choicesSheetName = 'choices') {
+        // Validate required sheets
+        this.validateRequiredSheets(hasSurveySheet, hasChoicesSheet);
+        // Validate survey sheet columns
+        if (hasSurveySheet && surveyData.length > 0) {
+            this.validateSurveySheetColumns(surveyData, surveySheetName);
+        }
+        // Validate choices sheet columns
+        if (hasChoicesSheet && choicesData.length > 0) {
+            this.validateChoicesSheetColumns(choicesData, choicesSheetName);
+        }
+    }
+}

package/dist/utils/helpers.js

@@ -0,0 +1,42 @@
+/**
+ * Deep merge objects - merges properties from source into target recursively
+ */
+export function deepMerge(target, ...sources) {
+    if (!sources.length)
+        return target;
+    const source = sources.shift();
+    if (isObject(target) && isObject(source)) {
+        for (const key in source) {
+            if (isObject(source[key])) {
+                if (!target[key]) {
+                    target[key] = {};
+                }
+                deepMerge(target[key], source[key]);
+            }
+            else {
+                target[key] = source[key];
+            }
+        }
+    }
+    return deepMerge(target, ...sources);
+}
+function isObject(item) {
+    return item !== null && typeof item === 'object' && !Array.isArray(item);
+}
+/**
+ * Sanitize field names for LimeSurvey compatibility
+ * Removes underscores/hyphens and truncates to 20 characters (LimeSurvey question title limit)
+ */
+export function sanitizeFieldName(name) {
+    let result = name;
+    // Always remove underscores and hyphens (LimeSurvey only allows alphanumeric)
+    result = result.replace(/[_-]/g, '');
+    // Limit to 20 characters (LimeSurvey questions.title field limit)
+    const maxLength = 20;
+    if (result.length > maxLength) {
+        const truncated = result.substring(0, maxLength);
+        console.warn(`Field name "${name}" exceeds maximum length of ${maxLength} characters and will be truncated to "${truncated}"`);
+        return truncated;
+    }
+    return result;
+}