xlsform2lstsv 0.2.3 → 0.4.0

package/dist/xlsformConverter.js

@@ -3,7 +3,9 @@ import { convertRelevance, convertConstraint, xpathToLimeSurvey } from './conver
 import { FieldSanitizer } from './processors/FieldSanitizer.js';
 import { TSVGenerator } from './processors/TSVGenerator.js';
 import { TypeMapper } from './processors/TypeMapper.js';
+import { deduplicateNames } from './utils/helpers.js';
 import { getBaseLanguage } from './utils/languageUtils.js';
+import { markdownToHtml } from './utils/markdownRenderer.js';
 // Metadata types that should be silently skipped (no visual representation)
 const SKIP_TYPES = [
     'start', 'end', 'today', 'deviceid', 'username',
@@ -22,14 +24,18 @@ const UNIMPLEMENTED_TYPES = [
     'begin_repeat',
     'end_repeat'
 ];
-// Appearances handled without warning: label, list-nolabel (matrix), multiline (→T), likert (no-op), field-list (no-op with format=A)
+// Appearances handled without warning: label, list-nolabel (matrix), multiline (→T), minimal (→!), likert (no-op), field-list (no-op: groups already map to LS groups; format=G used when style=pages)
 const UNSUPPORTED_APPEARANCES = [
-    'minimal', 'quick', 'no-calendar', 'month-year', 'year',
+    'quick', 'no-calendar', 'month-year', 'year',
     'horizontal-compact', 'horizontal', 'compact', 'quickcompact',
     'table-list', 'signature', 'draw', 'map', 'quick map',
 ];
 export class XLSFormToTSVConverter {
     constructor(config) {
+        this.welcomeNote = null;
+        this.endNote = null;
+        this.messageOnlyGroups = new Set();
+        this.surveyDataCache = [];
         this.configManager = new ConfigManager(config);
         this.configManager.validateConfig();
         this.fieldSanitizer = new FieldSanitizer();
@@ -53,6 +59,46 @@ export class XLSFormToTSVConverter {
         this.questionToListMap = new Map();
         this.questionBaseTypeMap = new Map();
     }
+    // ── Row helpers ──────────────────────────────────────────────────────
+    /**
+     * Build a TSVRowData with sensible defaults. Only `class` and `name` are required;
+     * all other fields default to empty strings (relevance defaults to '1').
+     */
+    row(fields) {
+        return {
+            'type/scale': '',
+            relevance: '1',
+            text: '',
+            help: '',
+            language: this.baseLanguage,
+            validation: '',
+            em_validation_q: '',
+            mandatory: '',
+            other: '',
+            default: '',
+            same_default: '',
+            ...fields,
+        };
+    }
+    /**
+     * Emit a buffered row for each available language. The callback receives the
+     * language code and returns the language-varying fields; common fields like
+     * `class` and `name` should be included in the callback return.
+     */
+    emitForEachLanguage(buildRow, target) {
+        if (!target)
+            target = 'buffer';
+        for (const lang of this.availableLanguages) {
+            const row = this.row({ language: lang, ...buildRow(lang) });
+            if (target === 'direct') {
+                this.tsvGenerator.addRow(row);
+            }
+            else {
+                this.bufferRow(row);
+            }
+        }
+    }
+    // ── Public API ───────────────────────────────────────────────────────
     /**
      * Get the current configuration
      */
@@ -79,14 +125,32 @@ export class XLSFormToTSVConverter {
         this.inMatrix = false;
         this.matrixListName = null;
         this.groupContentBuffer = [];
+        this.welcomeNote = null;
+        this.endNote = null;
+        // Pre-scan for welcome/end notes (must happen before group identification)
+        const config = this.configManager.getConfig();
+        for (const row of surveyData) {
+            const type = (row.type || '').trim();
+            const name = (row.name || '').trim().toLowerCase();
+            if (config.convertWelcomeNote && type === 'note' && name === 'welcome')
+                this.welcomeNote = row;
+            if (config.convertEndNote && type === 'note' && name === 'end')
+                this.endNote = row;
+        }
         // Pre-scan to identify parent-only groups (no direct questions, only child groups)
         this.parentOnlyGroups = this.identifyParentOnlyGroups(surveyData);
+        // Pre-scan to identify groups whose only content is a welcome/end note
+        this.messageOnlyGroups = this.identifyMessageOnlyGroups(surveyData);
+        // Cache survey data for pattern detection
+        this.surveyDataCache = surveyData;
         // Set base language from settings first
         this.baseLanguage = getBaseLanguage(settingsData[0] || {});
         // Detect available languages from survey data (will use baseLanguage for ordering)
         this.detectAvailableLanguages(surveyData, choicesData, settingsData);
         // Build choices map
         this.buildChoicesMap(choicesData);
+        // Pre-scan: register all field names to detect and resolve collisions
+        this.registerFieldNames(surveyData);
         // Build answer code and question-to-list maps for relevance rewriting
         this.buildAnswerCodeMap();
         this.buildQuestionToListMap(surveyData);
@@ -113,6 +177,55 @@ export class XLSFormToTSVConverter {
         // Generate TSV
         return this.tsvGenerator.generateTSV();
     }
+    // ── Other question pattern detection ─────────────────────────────────
+    /**
+     * Check if a question has a corresponding "_other" question with relevance targeting its "other" option.
+     * Returns true if pattern is found, and also removes the "other" choice from the choices list if present.
+     */
+    hasOtherQuestionPattern(currentRow, surveyData) {
+        const currentName = currentRow.name?.trim();
+        if (!currentName)
+            return false;
+        const otherQuestionName = `${currentName}_other`;
+        const sanitizedCurrentName = this.sanitizeName(currentName);
+        for (const row of surveyData) {
+            if (row.name?.trim() !== otherQuestionName || !row.relevant)
+                continue;
+            const relevance = row.relevant.trim();
+            // Pattern: ${question_name} = 'other' or ${question_name} == 'other'
+            const patterns = [currentName, sanitizedCurrentName].flatMap(n => [
+                new RegExp(`\\$\\{${n}\\}.*=.*['"]other['"]`),
+                new RegExp(`\\$\\{${n}\\}.*==.*['"]other['"]`),
+            ]);
+            if (patterns.some(p => p.test(relevance))) {
+                const xfTypeInfo = this.parseType(currentRow.type || '');
+                this.removeOtherChoiceFromList(currentRow, xfTypeInfo);
+                return true;
+            }
+        }
+        return false;
+    }
+    /**
+     * Remove the "other" choice from the choices list for a question.
+     * Prevents duplicate "other" options when using the _other question pattern.
+     */
+    removeOtherChoiceFromList(row, typeInfo) {
+        if (!typeInfo.listName)
+            return;
+        const choices = this.choicesMap.get(typeInfo.listName);
+        if (!choices)
+            return;
+        const otherNames = new Set(['other', '_other', 'other_option', 'other_choice']);
+        const filteredChoices = choices.filter(choice => {
+            const choiceName = choice.name?.trim().toLowerCase() || '';
+            return !otherNames.has(choiceName);
+        });
+        if (filteredChoices.length < choices.length) {
+            console.log(`Removed "other" choice(s) from list "${typeInfo.listName}" for question "${row.name}" when using _other question pattern`);
+            this.choicesMap.set(typeInfo.listName, filteredChoices);
+        }
+    }
+    // ── Language detection ────────────────────────────────────────────────
     detectAvailableLanguages(surveyData, choicesData, settingsData) {
         const languageCodes = new Set();
         // Check survey data for language codes
@@ -120,24 +233,36 @@ export class XLSFormToTSVConverter {
             if (row._languages) {
                 row._languages.forEach((lang) => languageCodes.add(lang));
             }
+            else {
+                for (const field of [row.label, row.hint]) {
+                    if (typeof field === 'object' && field !== null) {
+                        for (const lang of Object.keys(field))
+                            languageCodes.add(lang);
+                    }
+                }
+            }
         }
         // Check choices data for language codes
         for (const row of choicesData) {
             if (row._languages) {
                 row._languages.forEach((lang) => languageCodes.add(lang));
             }
+            else {
+                if (typeof row.label === 'object' && row.label !== null) {
+                    for (const lang of Object.keys(row.label))
+                        languageCodes.add(lang);
+                }
+            }
         }
         // Check settings data for language-specific fields
         const settings = settingsData[0] || {};
         for (const [, value] of Object.entries(settings)) {
             if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
-                // This looks like a language-specific field (e.g., {en: '...', es: '...'})
                 for (const lang of Object.keys(value)) {
                     languageCodes.add(lang);
                 }
             }
         }
-        // If no language-specific columns detected, use single language mode 
         // Only use multiple languages if we actually have language-specific data
         const hasLanguageSpecificData = surveyData.some(row => row._languages ||
             (typeof row.label === 'object' && row.label !== null) ||
@@ -145,7 +270,6 @@ export class XLSFormToTSVConverter {
             (typeof row.label === 'object' && row.label !== null)) || Object.values(settings).some(value => typeof value === 'object' && value !== null && !Array.isArray(value));
         if (hasLanguageSpecificData && languageCodes.size > 0) {
             const languagesArray = Array.from(languageCodes);
-            // Ensure default language comes first, then sort the rest alphabetically
             const defaultLang = this.baseLanguage;
             this.availableLanguages = [
                 defaultLang,
@@ -156,6 +280,7 @@ export class XLSFormToTSVConverter {
             this.availableLanguages = ['en'];
         }
     }
+    // ── Map building ─────────────────────────────────────────────────────
     buildChoicesMap(choices) {
         for (const choice of choices) {
             const listName = choice.list_name;
@@ -171,30 +296,15 @@ export class XLSFormToTSVConverter {
         this.answerCodeMap = new Map();
         for (const [listName, choices] of this.choicesMap) {
             const codeMap = new Map();
-            const sanitized = [];
-            for (const choice of choices) {
-                const raw = choice.name?.trim() || '';
-                sanitized.push(raw ? this.sanitizeAnswerCode(raw) : '');
-            }
-            const used = new Set();
-            for (let i = 0; i < sanitized.length; i++) {
-                if (!sanitized[i])
-                    continue;
-                let name = sanitized[i];
-                if (used.has(name)) {
-                    let counter = 1;
-                    let candidate;
-                    do {
-                        const suffix = String(counter);
-                        candidate = name.substring(0, 5 - suffix.length) + suffix;
-                        counter++;
-                    } while (used.has(candidate));
-                    sanitized[i] = candidate;
-                }
-                used.add(sanitized[i]);
+            const sanitized = choices.map(c => {
+                const raw = c.name?.trim() || '';
+                return raw ? this.sanitizeAnswerCode(raw) : '';
+            });
+            const deduplicated = deduplicateNames(sanitized, 5);
+            for (let i = 0; i < choices.length; i++) {
                 const originalName = choices[i].name?.trim() || '';
-                if (originalName) {
-                    codeMap.set(originalName, sanitized[i]);
+                if (originalName && deduplicated[i]) {
+                    codeMap.set(originalName, deduplicated[i]);
                 }
             }
             this.answerCodeMap.set(listName, codeMap);
@@ -212,153 +322,54 @@ export class XLSFormToTSVConverter {
             }
         }
     }
-    addDefaultGroup() {
-        // Add a default group for surveys without explicit groups
-        const defaults = this.configManager.getDefaults();
-        const groupName = defaults.groupName;
-        this.currentGroup = groupName;
-        this.tsvGenerator.addRow({
-            class: 'G',
-            'type/scale': '',
-            name: groupName,
-            relevance: '1',
-            text: groupName,
-            help: '',
-            language: defaults.language,
-            validation: '',
-            em_validation_q: '',
-            mandatory: '',
-            other: '',
-            default: '',
-            same_default: ''
-        });
-        this.groupSeq++;
-    }
-    addAutoGroupForOrphans() {
-        const groupName = `G${this.groupSeq}`;
-        this.groupSeq++;
-        this.currentGroup = groupName;
-        const groupSeqKey = String(this.groupSeq);
-        for (const lang of this.availableLanguages) {
-            this.tsvGenerator.addRow({
-                class: 'G',
-                'type/scale': groupSeqKey,
-                name: groupName,
-                relevance: '1',
-                text: groupName,
-                help: '',
-                language: lang,
-                validation: '',
-                em_validation_q: '',
-                mandatory: '',
-                other: '',
-                default: '',
-                same_default: ''
-            });
-        }
-    }
-    addSurveyRow(settings) {
-        // Add survey-level settings as individual S rows
-        // Each survey property gets its own row with name=property_name, text=value
-        const defaults = this.configManager.getDefaults();
-        const surveyTitle = settings.form_title || defaults.surveyTitle;
-        // Add base language (class S - survey settings)
-        this.tsvGenerator.addRow({
-            class: 'S',
-            'type/scale': '',
-            name: 'language',
-            relevance: '1',
-            text: this.baseLanguage,
-            help: '',
-            language: this.baseLanguage,
-            validation: '',
-            em_validation_q: '',
-            mandatory: '',
-            other: '',
-            default: '',
-            same_default: ''
-        });
-        // Add additional languages declaration (class S - survey settings)
-        // This tells LimeSurvey which additional languages should be available
-        if (this.availableLanguages.length > 1) {
-            const additionalLanguages = this.availableLanguages.filter(lang => lang !== this.baseLanguage).join(' ');
-            this.tsvGenerator.addRow({
-                class: 'S',
-                'type/scale': '',
-                name: 'additional_languages',
-                relevance: '1',
-                text: additionalLanguages,
-                help: '',
-                language: this.baseLanguage,
-                validation: '',
-                em_validation_q: '',
-                mandatory: '',
-                other: '',
-                default: '',
-                same_default: ''
-            });
-        }
-        // Set survey format to "All in one" (all groups/questions on one page)
-        this.tsvGenerator.addRow({
-            class: 'S',
-            'type/scale': '',
-            name: 'format',
-            relevance: '1',
-            text: 'A',
-            help: '',
-            language: this.baseLanguage,
-            validation: '',
-            em_validation_q: '',
-            mandatory: '',
-            other: '',
-            default: '',
-            same_default: ''
-        });
-        // First, add the default language row according to LimeSurvey spec
-        // This should be the first SL row for the default language
-        const defaultLanguage = this.baseLanguage;
-        // Add default language row (class SL - survey language settings)
-        this.tsvGenerator.addRow({
-            class: 'SL',
-            'type/scale': '',
-            name: 'surveyls_title',
-            relevance: '1',
-            text: this.getLanguageSpecificValue(settings.form_title, defaultLanguage) || surveyTitle,
-            help: '',
-            language: defaultLanguage,
-            validation: '',
-            em_validation_q: '',
-            mandatory: '',
-            other: '',
-            default: '',
-            same_default: ''
-        });
-        // Then add rows for all other available languages (after default language)
-        const otherLanguages = this.availableLanguages.filter(lang => lang !== defaultLanguage);
-        // Sort other languages alphabetically for consistency
-        otherLanguages.sort();
-        for (const lang of otherLanguages) {
-            this.tsvGenerator.addRow({
-                class: 'SL',
-                'type/scale': '',
-                name: 'surveyls_title',
-                relevance: '1',
-                text: this.getLanguageSpecificValue(settings.form_title, lang) || surveyTitle,
-                help: '',
-                language: lang,
-                validation: '',
-                em_validation_q: "",
-                mandatory: '',
-                other: '',
-                default: '',
-                same_default: ''
-            });
+    // ── Pre-scanning ─────────────────────────────────────────────────────
+    /**
+     * Pre-scan survey data to identify groups whose only direct content is a
+     * welcome or end note. These groups are silently suppressed.
+     */
+    identifyMessageOnlyGroups(surveyData) {
+        const messageOnly = new Set();
+        const stack = [];
+        const groupInfo = new Map();
+        for (const row of surveyData) {
+            const type = (row.type || '').trim();
+            const baseType = type.split(/\s+/)[0];
+            if (type === 'begin_group' || type === 'begin group') {
+                const name = (row.name || '').trim();
+                stack.push(name);
+                groupInfo.set(name, { hasMessageNote: false, hasOtherContent: false });
+            }
+            else if (type === 'end_group' || type === 'end group') {
+                const name = stack.pop();
+                if (name !== undefined) {
+                    const info = groupInfo.get(name);
+                    if (info && info.hasMessageNote && !info.hasOtherContent) {
+                        messageOnly.add(name);
+                    }
+                }
+            }
+            else if (type && stack.length > 0 && !SKIP_TYPES.includes(baseType)) {
+                const groupName = stack[stack.length - 1];
+                const info = groupInfo.get(groupName);
+                if (info) {
+                    const rowName = (row.name || '').trim().toLowerCase();
+                    const cfg = this.configManager.getConfig();
+                    const isMessageNote = type === 'note' && ((cfg.convertWelcomeNote && rowName === 'welcome') ||
+                        (cfg.convertEndNote && rowName === 'end'));
+                    if (isMessageNote) {
+                        info.hasMessageNote = true;
+                    }
+                    else {
+                        info.hasOtherContent = true;
+                    }
+                }
+            }
         }
+        return messageOnly;
     }
     /**
      * Pre-scan survey data to identify parent-only groups.
      * A parent-only group contains no direct questions — only child groups.
-     * These will be flattened into note questions in the first child group.
      */
     identifyParentOnlyGroups(surveyData) {
         const parentOnly = new Set();
@@ -379,7 +390,6 @@ export class XLSFormToTSVConverter {
                 }
             }
             else if (type && !SKIP_TYPES.includes(baseType)) {
-                // Any non-skip, non-group row counts as direct content
                 if (stack.length > 0) {
                     hasDirectContent.set(stack[stack.length - 1], true);
                 }
@@ -387,107 +397,50 @@ export class XLSFormToTSVConverter {
         }
         return parentOnly;
     }
-    async processRow(row) {
-        const xfType = (row.type || '').trim();
-        if (!xfType)
-            return;
-        // Check for unimplemented types (extract base type first, before any spaces)
-        const baseType = xfType.split(/\s+/)[0];
-        // Silently skip metadata types
-        if (SKIP_TYPES.includes(baseType)) {
-            return;
-        }
-        // Other unimplemented types throw errors
-        if (UNIMPLEMENTED_TYPES.includes(baseType)) {
-            throw new Error(`Unimplemented XLSForm type: '${baseType}'. This type is not currently supported.`);
-        }
-        if (xfType === 'begin_group' || xfType === 'begin group') {
-            this.flushMatrix();
-            const originalName = (row.name || '').trim();
-            const sanitizedName = originalName
-                ? this.sanitizeName(originalName)
-                : `G${this.groupSeq}`;
-            if (this.parentOnlyGroups.has(originalName)) {
-                // Parent-only group: save label as pending note, don't emit G row
-                this.groupStack.push({ originalName, sanitizedName, emittedAsGroup: false });
-                this.pendingGroupNotes.push(row);
-            }
-            else {
-                // Regular group: flush buffered content from previous group, then emit G row
-                this.groupStack.push({ originalName, sanitizedName, emittedAsGroup: true });
-                this.flushGroupContent();
-                await this.addGroup(row);
-                // Emit pending parent-only group notes as note questions in this group
-                await this.emitPendingGroupNotes();
-            }
-            return;
-        }
-        if (xfType === 'end_group' || xfType === 'end group') {
-            this.flushMatrix();
-            this.groupStack.pop();
-            // Restore currentGroup to nearest ancestor that was emitted as a group
-            this.currentGroup = null;
-            for (let i = this.groupStack.length - 1; i >= 0; i--) {
-                if (this.groupStack[i].emittedAsGroup) {
-                    this.currentGroup = this.groupStack[i].sanitizedName;
-                    break;
-                }
-            }
-            return;
-        }
-        // Auto-create a group for questions outside any explicit group.
-        // LimeSurvey requires every question to belong to a group.
-        if (this.currentGroup === null && this.groupStack.length === 0) {
-            this.flushGroupContent();
-            this.addAutoGroupForOrphans();
+    // ── Field name handling ──────────────────────────────────────────────
+    /**
+     * Pre-scan all survey rows and register every field name with the sanitizer,
+     * so that collisions after sanitization + truncation are detected early.
+     */
+    registerFieldNames(surveyData) {
+        this.fieldSanitizer.resetNames();
+        for (const row of surveyData) {
+            const type = (row.type || '').trim();
+            if (type === 'end_group' || type === 'end_repeat')
+                continue;
+            const name = row.name?.trim();
+            if (!name)
+                continue;
+            this.fieldSanitizer.sanitizeNameUnique(name);
         }
-        // Handle notes and questions
-        await this.addQuestion(row);
+    }
+    sanitizeName(name) {
+        const stripped = name.replace(/[_-]/g, '');
+        return this.fieldSanitizer.resolveStrippedName(stripped);
+    }
+    sanitizeAnswerCode(code) {
+        return this.fieldSanitizer.sanitizeAnswerCode(code);
     }
     /**
-     * Emit pending parent-only group labels as note questions (type X).
-     * Called after a child group's G row is emitted.
+     * Convert ${varname} references in text to LimeSurvey EM syntax {sanitizedname}.
      */
-    async emitPendingGroupNotes() {
-        for (const noteRow of this.pendingGroupNotes) {
-            const noteName = noteRow.name && noteRow.name.trim() !== ''
-                ? this.sanitizeName(noteRow.name.trim())
-                : `GN${this.questionSeq}`;
-            this.questionSeq++;
-            for (const lang of this.availableLanguages) {
-                this.bufferRow({
-                    class: 'Q',
-                    'type/scale': 'X',
-                    name: noteName,
-                    relevance: await this.convertRelevance(noteRow.relevant),
-                    text: this.getLanguageSpecificValue(noteRow.label, lang) || noteName,
-                    help: this.getLanguageSpecificValue(noteRow.hint, lang) || '',
-                    language: lang,
-                    validation: '',
-                    em_validation_q: '',
-                    mandatory: '',
-                    other: '',
-                    default: '',
-                    same_default: ''
-                });
-            }
-        }
-        this.pendingGroupNotes = [];
+    convertVariableReferences(text) {
+        return text.replace(/\$\{([^}]+)\}/g, (_, name) => {
+            const sanitized = this.sanitizeName(name);
+            return `{${sanitized}}`;
+        });
     }
+    // ── Label rendering ──────────────────────────────────────────────────
     getLanguageSpecificValue(value, languageCode) {
         if (!value)
             return undefined;
-        // If it's already a string, return it
-        if (typeof value === 'string') {
+        if (typeof value === 'string')
             return value;
-        }
-        // If it's an object with language codes, get the specific language
         if (typeof value === 'object' && value !== null) {
             const valueObj = value;
             if (languageCode in valueObj) {
                 return valueObj[languageCode];
             }
-            // If it doesn't have the specific language, try to get any available language
             for (const lang of this.availableLanguages) {
                 if (lang in valueObj)
                     return valueObj[lang];
@@ -495,34 +448,36 @@ export class XLSFormToTSVConverter {
         }
         return undefined;
     }
+    /**
+     * Resolve a multilingual label/hint value for the given language and optionally
+     * convert it from markdown to HTML. Falls back to `fallback` when no value is found.
+     */
+    renderLabel(value, lang, fallback = '') {
+        const raw = this.getLanguageSpecificValue(value, lang) || fallback;
+        return this.configManager.getConfig().convertMarkdown ? markdownToHtml(raw) : raw;
+    }
+    // ── Buffering / flushing ─────────────────────────────────────────────
     /**
      * Buffer a Q/SQ/A row for later language-grouped output.
      * LimeSurvey's TSV importer uses a question_order counter ($qseq) that gets
      * reset when it encounters a translation of a previously-seen question.
-     * With interleaved languages (Q de, Q en, Q de, Q en), the counter resets
-     * after each translation, giving all subsequent questions order=0.
-     * By outputting all base-language rows first, the counter increments correctly,
-     * and translation rows just look up their stored values.
+     * By outputting all base-language rows first, the counter increments correctly.
      */
     bufferRow(row) {
         this.groupContentBuffer.push(row);
     }
     /**
      * Flush buffered group content, outputting base language rows first,
-     * then each additional language. This preserves insertion order within
-     * each language while ensuring LimeSurvey's question_order counter
-     * increments correctly.
+     * then each additional language.
      */
     flushGroupContent() {
         if (this.groupContentBuffer.length === 0)
             return;
-        // Output base language rows first (preserving insertion order)
         for (const row of this.groupContentBuffer) {
             if (row.language === this.baseLanguage) {
                 this.tsvGenerator.addRow(row);
             }
         }
-        // Then output each additional language (preserving insertion order)
         for (const lang of this.availableLanguages) {
             if (lang === this.baseLanguage)
                 continue;
@@ -534,58 +489,172 @@ export class XLSFormToTSVConverter {
         }
         this.groupContentBuffer = [];
     }
-    sanitizeName(name) {
-        return this.fieldSanitizer.sanitizeName(name);
-    }
-    sanitizeAnswerCode(code) {
-        return this.fieldSanitizer.sanitizeAnswerCode(code);
+    // ── Survey settings (S/SL rows) ─────────────────────────────────────
+    addSurveyRow(settings) {
+        const defaults = this.configManager.getDefaults();
+        const surveyTitle = settings.form_title || defaults.surveyTitle;
+        // S rows: language, additional_languages, format
+        this.tsvGenerator.addRow(this.row({
+            class: 'S', name: 'language', text: this.baseLanguage,
+        }));
+        if (this.availableLanguages.length > 1) {
+            const additionalLanguages = this.availableLanguages.filter(lang => lang !== this.baseLanguage).join(' ');
+            this.tsvGenerator.addRow(this.row({
+                class: 'S', name: 'additional_languages', text: additionalLanguages,
+            }));
+        }
+        const surveyFormat = (settings.style || '').trim().toLowerCase() === 'pages' ? 'G' : 'A';
+        this.tsvGenerator.addRow(this.row({
+            class: 'S', name: 'format', text: surveyFormat,
+        }));
+        // SL rows: survey title + welcome/end text, base language first then others
+        const emitSLRows = (lang) => {
+            this.tsvGenerator.addRow(this.row({
+                class: 'SL', name: 'surveyls_title', language: lang,
+                text: this.renderLabel(settings.form_title, lang, surveyTitle),
+            }));
+            this.addSLMessageRows(lang);
+        };
+        emitSLRows(this.baseLanguage);
+        for (const lang of this.availableLanguages.filter(l => l !== this.baseLanguage).sort()) {
+            emitSLRows(lang);
+        }
     }
     /**
-     * Convert ${varname} references in text to LimeSurvey EM syntax {sanitizedname}.
+     * Emit surveyls_welcometext and surveyls_endtext SL rows for a given language.
      */
-    convertVariableReferences(text) {
-        return text.replace(/\$\{([^}]+)\}/g, (_, name) => {
-            const sanitized = name.replace(/[_-]/g, '');
-            return `{${sanitized}}`;
-        });
+    addSLMessageRows(lang) {
+        if (this.welcomeNote) {
+            this.tsvGenerator.addRow(this.row({
+                class: 'SL', name: 'surveyls_welcometext', language: lang,
+                text: this.renderLabel(this.welcomeNote.label, lang),
+            }));
+        }
+        if (this.endNote) {
+            this.tsvGenerator.addRow(this.row({
+                class: 'SL', name: 'surveyls_endtext', language: lang,
+                text: this.renderLabel(this.endNote.label, lang),
+            }));
+        }
     }
-    /**
-     * Transpile an XLSForm calculation expression to a LimeSurvey EM expression.
-     */
-    async convertCalculation(calculation) {
-        return await xpathToLimeSurvey(calculation);
+    // ── Group handling ───────────────────────────────────────────────────
+    addDefaultGroup() {
+        const defaults = this.configManager.getDefaults();
+        const groupName = defaults.groupName;
+        this.currentGroup = groupName;
+        this.tsvGenerator.addRow(this.row({
+            class: 'G', name: groupName, text: groupName, language: defaults.language,
+        }));
+        this.groupSeq++;
+    }
+    addAutoGroupForOrphans() {
+        const groupName = `G${this.groupSeq}`;
+        this.groupSeq++;
+        this.currentGroup = groupName;
+        const groupSeqKey = String(this.groupSeq);
+        this.emitForEachLanguage(() => ({
+            class: 'G', 'type/scale': groupSeqKey, name: groupName, text: groupName,
+        }), 'direct');
     }
     async addGroup(row) {
-        // Auto-generate name if missing (matches LimeSurvey behavior)
         const groupName = row.name && row.name.trim() !== ''
             ? this.sanitizeName(row.name.trim())
             : `G${this.groupSeq}`;
         this.groupSeq++;
         this.currentGroup = groupName;
-        // Groups support relevance but not validation.
-        // LimeSurvey's TSV importer matches group translations across languages
-        // using the type/scale column as a stable group sequence key. Without it,
-        // an auto-counter resets on each language change and mismatches groups.
-        // We set type/scale to the group sequence number to ensure correct matching.
+        // type/scale is used as a stable group sequence key for LimeSurvey's TSV importer
+        // to correctly match group translations across languages.
         const groupSeqKey = String(this.groupSeq);
-        for (const lang of this.availableLanguages) {
-            this.tsvGenerator.addRow({
-                class: 'G',
-                'type/scale': groupSeqKey,
-                name: groupName,
-                relevance: await this.convertRelevance(row.relevant),
-                text: this.getLanguageSpecificValue(row.label, lang) || groupName,
-                help: this.getLanguageSpecificValue(row.hint, lang) || '',
-                language: lang,
-                validation: '',
-                em_validation_q: "",
-                mandatory: '',
-                other: '',
-                default: '',
-                same_default: ''
-            });
+        const relevance = await this.convertRelevance(row.relevant);
+        this.emitForEachLanguage(lang => ({
+            class: 'G', 'type/scale': groupSeqKey,
+            name: this.renderLabel(row.label, lang, groupName),
+            relevance,
+            text: this.renderLabel(row.hint, lang),
+        }), 'direct');
+    }
+    /**
+     * Emit pending parent-only group labels as note questions (type X).
+     */
+    async emitPendingGroupNotes() {
+        for (const noteRow of this.pendingGroupNotes) {
+            const noteName = noteRow.name && noteRow.name.trim() !== ''
+                ? this.sanitizeName(noteRow.name.trim())
+                : `GN${this.questionSeq}`;
+            this.questionSeq++;
+            const relevance = await this.convertRelevance(noteRow.relevant);
+            this.emitForEachLanguage(lang => ({
+                class: 'Q', 'type/scale': 'X', name: noteName,
+                relevance,
+                text: this.renderLabel(noteRow.label, lang, noteName),
+                help: this.renderLabel(noteRow.hint, lang),
+            }));
+        }
+        this.pendingGroupNotes = [];
+    }
+    // ── Row processing ───────────────────────────────────────────────────
+    async processRow(row) {
+        const xfType = (row.type || '').trim();
+        if (!xfType)
+            return;
+        const baseType = xfType.split(/\s+/)[0];
+        // Silently skip metadata types
+        if (SKIP_TYPES.includes(baseType))
+            return;
+        // Skip notes that have been promoted to welcome/end messages
+        if (xfType === 'note') {
+            const name = (row.name || '').trim().toLowerCase();
+            const cfg = this.configManager.getConfig();
+            if (cfg.convertWelcomeNote && name === 'welcome')
+                return;
+            if (cfg.convertEndNote && name === 'end')
+                return;
+        }
+        // Other unimplemented types throw errors
+        if (UNIMPLEMENTED_TYPES.includes(baseType)) {
+            throw new Error(`Unimplemented XLSForm type: '${baseType}'. This type is not currently supported.`);
+        }
+        if (xfType === 'begin_group' || xfType === 'begin group') {
+            this.flushMatrix();
+            const originalName = (row.name || '').trim();
+            const sanitizedName = originalName
+                ? this.sanitizeName(originalName)
+                : `G${this.groupSeq}`;
+            if (this.messageOnlyGroups.has(originalName)) {
+                this.groupStack.push({ originalName, sanitizedName, emittedAsGroup: false });
+            }
+            else if (this.parentOnlyGroups.has(originalName)) {
+                this.groupStack.push({ originalName, sanitizedName, emittedAsGroup: false });
+                this.pendingGroupNotes.push(row);
+            }
+            else {
+                this.groupStack.push({ originalName, sanitizedName, emittedAsGroup: true });
+                this.flushGroupContent();
+                await this.addGroup(row);
+                await this.emitPendingGroupNotes();
+            }
+            return;
+        }
+        if (xfType === 'end_group' || xfType === 'end group') {
+            this.flushMatrix();
+            this.groupStack.pop();
+            this.currentGroup = null;
+            for (let i = this.groupStack.length - 1; i >= 0; i--) {
+                if (this.groupStack[i].emittedAsGroup) {
+                    this.currentGroup = this.groupStack[i].sanitizedName;
+                    break;
+                }
+            }
+            return;
+        }
+        // Auto-create a group for questions outside any explicit group.
+        if (this.currentGroup === null && this.groupStack.length === 0) {
+            this.flushGroupContent();
+            this.addAutoGroupForOrphans();
         }
+        await this.addQuestion(row);
     }
+    // ── Question emission ────────────────────────────────────────────────
     async addQuestion(row) {
         const xfTypeInfo = this.parseType(row.type || '');
         const appearance = typeof row['appearance'] === 'string' ? row['appearance'].trim() : '';
@@ -604,14 +673,14 @@ export class XLSFormToTSVConverter {
         this.flushMatrix();
         // Warn on unsupported appearances
         if (appearance) {
-            const parts = appearance.split(/\s+/);
-            for (const part of parts) {
-                if (UNSUPPORTED_APPEARANCES.includes(part)) {
+            for (const part of appearance.split(/\s+/)) {
+                const isUnsupported = UNSUPPORTED_APPEARANCES.includes(part)
+                    || (part === 'minimal' && xfTypeInfo.base !== 'select_one');
+                if (isUnsupported) {
                     console.warn(`Unsupported appearance "${part}" on question "${row.name}" will be ignored`);
                 }
             }
         }
-        // Auto-generate name if missing (matches LimeSurvey behavior)
         const questionName = row.name && row.name.trim() !== ''
             ? this.sanitizeName(row.name.trim())
             : `Q${this.questionSeq}`;
@@ -620,48 +689,44 @@ export class XLSFormToTSVConverter {
         // Appearance-based type overrides
         if (appearance) {
             const parts = appearance.split(/\s+/);
-            // multiline text → Long free text (T)
             if (parts.includes('multiline') && (xfTypeInfo.base === 'text' || xfTypeInfo.base === 'string')) {
                 lsType.type = 'T';
             }
+            if (parts.includes('minimal') && xfTypeInfo.base === 'select_one') {
+                lsType.type = '!';
+            }
         }
-        // Notes have special handling
         const isNote = xfTypeInfo.base === 'note';
         const isCalculate = xfTypeInfo.base === 'calculate';
-        // For calculate type, transpile the calculation expression to EM syntax
+        const isNoteOrCalc = isNote || isCalculate;
         let calculationExpr = '';
         if (isCalculate && row.calculation) {
             calculationExpr = await this.convertCalculation(row.calculation);
         }
-        // Add main question for each language
-        for (const lang of this.availableLanguages) {
+        const relevance = await this.convertRelevance(row.relevant);
+        const emValidation = isNoteOrCalc ? '' : await convertConstraint(row.constraint || '');
+        const mandatory = isNoteOrCalc ? '' : (row.required === 'yes' || row.required === 'true' ? 'Y' : '');
+        const otherPattern = this.configManager.getConfig().convertOtherPattern ? this.hasOtherQuestionPattern(row, this.surveyDataCache) : false;
+        const other = isNoteOrCalc ? '' : ((lsType.other || otherPattern) ? 'Y' : '');
+        const defaultVal = isNoteOrCalc ? '' : (row.default || '');
+        this.emitForEachLanguage(lang => {
             let text;
             if (isCalculate) {
-                // Equation question: the EM expression wrapped in {} IS the question text
                 text = `{${calculationExpr}}`;
             }
             else {
-                text = this.getLanguageSpecificValue(row.label, lang) || questionName;
+                text = this.renderLabel(row.label, lang, questionName);
             }
-            // Convert ${var} references to EM {var} syntax in text and help
             text = this.convertVariableReferences(text);
-            const help = this.convertVariableReferences(this.getLanguageSpecificValue(row.hint, lang) || '');
-            this.bufferRow({
-                class: 'Q',
-                'type/scale': isNote ? 'X' : lsType.type,
-                name: questionName,
-                relevance: await this.convertRelevance(row.relevant),
-                text,
-                help,
-                language: lang,
-                validation: "",
-                em_validation_q: (isNote || isCalculate) ? "" : await convertConstraint(row.constraint || ""),
-                mandatory: (isNote || isCalculate) ? '' : (row.required === 'yes' || row.required === 'true' ? 'Y' : ''),
-                other: (isNote || isCalculate) ? '' : (lsType.other ? 'Y' : ''),
-                default: (isNote || isCalculate) ? '' : (row.default || ''),
-                same_default: ''
-            });
-        }
+            const help = this.convertVariableReferences(this.renderLabel(row.hint, lang));
+            return {
+                class: 'Q', 'type/scale': isNote ? 'X' : lsType.type,
+                name: questionName, relevance, text, help,
+                em_validation_q: emValidation, mandatory, other,
+                default: defaultVal, same_default: '',
+                hidden: isCalculate ? '1' : '',
+            };
+        });
         // Reset answer sequence for this question
         this.answerSeq = 0;
         this.subquestionSeq = 0;
@@ -670,6 +735,7 @@ export class XLSFormToTSVConverter {
             this.addAnswers(xfTypeInfo, lsType);
         }
     }
+    // ── Matrix questions ─────────────────────────────────────────────────
     async addMatrixHeader(row, xfTypeInfo) {
         const questionName = row.name && row.name.trim() !== ''
             ? this.sanitizeName(row.name.trim())
@@ -678,47 +744,27 @@ export class XLSFormToTSVConverter {
         this.inMatrix = true;
         this.matrixListName = xfTypeInfo.listName;
         this.subquestionSeq = 0;
-        // Emit Q row with type F (Array)
-        for (const lang of this.availableLanguages) {
-            this.bufferRow({
-                class: 'Q',
-                'type/scale': 'F',
-                name: questionName,
-                relevance: await this.convertRelevance(row.relevant),
-                text: this.getLanguageSpecificValue(row.label, lang) || questionName,
-                help: this.getLanguageSpecificValue(row.hint, lang) || '',
-                language: lang,
-                validation: '',
-                em_validation_q: '',
-                mandatory: row.required === 'yes' || row.required === 'true' ? 'Y' : '',
-                other: '',
-                default: '',
-                same_default: ''
-            });
-        }
+        const relevance = await this.convertRelevance(row.relevant);
+        const mandatory = row.required === 'yes' || row.required === 'true' ? 'Y' : '';
+        this.emitForEachLanguage(lang => ({
+            class: 'Q', 'type/scale': 'F', name: questionName,
+            relevance, mandatory,
+            text: this.renderLabel(row.label, lang, questionName),
+            help: this.renderLabel(row.hint, lang),
+        }));
     }
     async addMatrixSubquestion(row) {
         const sqName = row.name && row.name.trim() !== ''
             ? this.sanitizeName(row.name.trim())
             : `SQ${this.subquestionSeq}`;
         this.subquestionSeq++;
-        for (const lang of this.availableLanguages) {
-            this.bufferRow({
-                class: 'SQ',
-                'type/scale': '',
-                name: sqName,
-                relevance: await this.convertRelevance(row.relevant),
-                text: this.getLanguageSpecificValue(row.label, lang) || sqName,
-                help: '',
-                language: lang,
-                validation: '',
-                em_validation_q: '',
-                mandatory: row.required === 'yes' || row.required === 'true' ? 'Y' : '',
-                other: '',
-                default: '',
-                same_default: ''
-            });
-        }
+        const relevance = await this.convertRelevance(row.relevant);
+        const mandatory = row.required === 'yes' || row.required === 'true' ? 'Y' : '';
+        this.emitForEachLanguage(lang => ({
+            class: 'SQ', name: sqName,
+            relevance, mandatory,
+            text: this.renderLabel(row.label, lang, sqName),
+        }));
     }
     flushMatrix() {
         if (!this.inMatrix || !this.matrixListName) {
@@ -733,97 +779,51 @@ export class XLSFormToTSVConverter {
                 const choiceName = choice.name && choice.name.trim() !== ''
                     ? this.sanitizeAnswerCode(choice.name.trim())
                     : `A${seq++}`;
-                for (const lang of this.availableLanguages) {
-                    this.bufferRow({
-                        class: 'A',
-                        'type/scale': '',
-                        name: choiceName,
-                        relevance: '',
-                        text: this.getLanguageSpecificValue(choice.label, lang) || choiceName,
-                        help: '',
-                        language: lang,
-                        validation: '',
-                        em_validation_q: '',
-                        mandatory: '',
-                        other: '',
-                        default: '',
-                        same_default: ''
-                    });
-                }
+                this.emitForEachLanguage(lang => ({
+                    class: 'A', name: choiceName, relevance: '',
+                    text: this.renderLabel(choice.label, lang, choiceName),
+                }));
             }
         }
         this.inMatrix = false;
         this.matrixListName = null;
     }
-    parseType(typeStr) {
-        return this.typeMapper.parseType(typeStr);
-    }
-    mapType(xfTypeInfo) {
-        return this.typeMapper.mapType(xfTypeInfo);
-    }
+    // ── Answer emission ──────────────────────────────────────────────────
     addAnswers(xfTypeInfo, lsType) {
         const choices = this.choicesMap.get(xfTypeInfo.listName);
         if (!choices) {
             console.warn(`Choice list not found: ${xfTypeInfo.listName}`);
             return;
         }
-        // Use the answer class from the type mapping
         const answerClass = lsType.answerClass || (xfTypeInfo.base === 'select_multiple' ? 'SQ' : 'A');
-        // Pre-compute sanitized choice names
-        const choiceNames = [];
-        for (const choice of choices) {
+        // Pre-compute and deduplicate sanitized choice names
+        const rawNames = choices.map(choice => {
             const rawName = choice.name && choice.name.trim() !== '' ? choice.name.trim() : '';
-            choiceNames.push(rawName
+            return rawName
                 ? this.sanitizeAnswerCode(rawName)
-                : (answerClass === 'SQ' ? `SQ${this.subquestionSeq++}` : `A${this.answerSeq++}`));
-        }
-        // Resolve duplicate names by appending a counter suffix
-        const usedNames = new Set();
-        for (let i = 0; i < choiceNames.length; i++) {
-            let name = choiceNames[i];
-            if (usedNames.has(name)) {
-                let counter = 1;
-                let candidate;
-                do {
-                    const suffix = String(counter);
-                    candidate = name.substring(0, 5 - suffix.length) + suffix;
-                    counter++;
-                } while (usedNames.has(candidate));
-                console.warn(`Duplicate answer code "${name}" resolved to "${candidate}"`);
-                choiceNames[i] = candidate;
-            }
-            usedNames.add(choiceNames[i]);
+                : (answerClass === 'SQ' ? `SQ${this.subquestionSeq++}` : `A${this.answerSeq++}`);
+        });
+        const choiceNames = deduplicateNames(rawNames, 5);
+        for (let i = 0; i < rawNames.length; i++) {
+            if (choiceNames[i] !== rawNames[i]) {
+                console.warn(`Duplicate answer code "${rawNames[i]}" resolved to "${choiceNames[i]}"`);
+            }
         }
         for (let i = 0; i < choices.length; i++) {
             const choice = choices[i];
             const choiceName = choiceNames[i];
-            // Add answer for each language
-            for (const lang of this.availableLanguages) {
-                this.bufferRow({
-                    class: answerClass,
-                    'type/scale': '',
-                    name: choiceName,
-                    relevance: choice.filter
-                        ? `({${this.currentGroup || 'parent'}} == "${choice.filter}")`
-                        : '',
-                    text: this.getLanguageSpecificValue(choice.label, lang) || choiceName,
-                    help: '',
-                    language: lang,
-                    validation: '',
-                    em_validation_q: '',
-                    mandatory: '',
-                    other: '',
-                    default: '',
-                    same_default: ''
-                });
-            }
+            this.emitForEachLanguage(lang => ({
+                class: answerClass, name: choiceName, relevance: '',
+                ...(choice.filter ? { relevance: `({${this.currentGroup || 'parent'}} == "${choice.filter}")` } : {}),
+                text: this.renderLabel(choice.label, lang, choiceName),
+            }));
         }
     }
+    // ── Expression transpilation ─────────────────────────────────────────
     lookupAnswerCode(fieldName, choiceValue) {
-        // Truncate to 20 chars to match converter's field sanitization
-        // (the transpiler only removes _/- but doesn't truncate)
-        const truncated = fieldName.length > 20 ? fieldName.substring(0, 20) : fieldName;
-        const listName = this.questionToListMap.get(truncated);
+        const stripped = fieldName.replace(/[_-]/g, '');
+        const resolved = this.fieldSanitizer.resolveStrippedName(stripped);
+        const listName = this.questionToListMap.get(resolved);
         if (!listName)
             return { code: choiceValue, listName: undefined };
         const codeMap = this.answerCodeMap.get(listName);
@@ -831,23 +831,38 @@ export class XLSFormToTSVConverter {
             return { code: choiceValue, listName };
         return { code: codeMap.get(choiceValue) ?? choiceValue, listName };
     }
-    async convertRelevance(relevant) {
-        if (!relevant)
-            return '1';
-        const ctx = {
+    buildTranspilerContext() {
+        return {
             lookupAnswerCode: (fieldName, choiceValue) => {
                 return this.lookupAnswerCode(fieldName, choiceValue).code;
             },
+            getTruncatedFieldName: (fieldName) => {
+                return this.fieldSanitizer.resolveStrippedName(fieldName);
+            },
             buildSelectedExpr: (fieldName, choiceValue) => {
-                const truncated = fieldName.length > 20 ? fieldName.substring(0, 20) : fieldName;
+                const resolved = this.fieldSanitizer.resolveStrippedName(fieldName);
                 const { code } = this.lookupAnswerCode(fieldName, choiceValue);
-                const baseType = this.questionBaseTypeMap.get(truncated);
+                const baseType = this.questionBaseTypeMap.get(resolved);
                 if (baseType === 'select_multiple') {
-                    return `(${truncated}_${code}.NAOK == "Y")`;
+                    return `(${resolved}_${code}.NAOK == 'Y')`;
                 }
-                return `(${truncated}.NAOK=="${code}")`;
+                return `(${resolved}.NAOK=='${code}')`;
             },
         };
-        return await convertRelevance(relevant, ctx);
+    }
+    async convertRelevance(relevant) {
+        if (!relevant)
+            return '1';
+        return await convertRelevance(relevant, this.buildTranspilerContext());
+    }
+    async convertCalculation(calculation) {
+        return await xpathToLimeSurvey(calculation, this.buildTranspilerContext());
+    }
+    // ── Type helpers ─────────────────────────────────────────────────────
+    parseType(typeStr) {
+        return this.typeMapper.parseType(typeStr);
+    }
+    mapType(xfTypeInfo) {
+        return this.typeMapper.mapType(xfTypeInfo);
     }
 }