pgsql-deparser 17.4.1 → 17.5.0

package/deparser.js

@@ -867,13 +867,16 @@ class Deparser {
             formatStr = '(%s)';
         }
         const boolContext = { ...context, bool: true };
+        // explanation of our syntax/fix below:
+        // return formatStr.replace('%s', andArgs); // ❌ Interprets $ as special syntax
+        // return formatStr.replace('%s', () => andArgs); // ✅ Function callback prevents interpretation
         switch (boolop) {
             case 'AND_EXPR':
                 const andArgs = args.map(arg => this.visit(arg, boolContext)).join(' AND ');
-                return formatStr.replace('%s', andArgs);
+                return formatStr.replace('%s', () => andArgs);
             case 'OR_EXPR':
                 const orArgs = args.map(arg => this.visit(arg, boolContext)).join(' OR ');
-                return formatStr.replace('%s', orArgs);
+                return formatStr.replace('%s', () => orArgs);
             case 'NOT_EXPR':
                 return `NOT (${this.visit(args[0], context)})`;
             default:
@@ -1132,23 +1135,23 @@ class Deparser {
         else if (nodeAny.sval !== undefined) {
             if (typeof nodeAny.sval === 'object' && nodeAny.sval !== null) {
                 if (nodeAny.sval.sval !== undefined) {
-                    return quote_utils_1.QuoteUtils.escape(nodeAny.sval.sval);
+                    return quote_utils_1.QuoteUtils.formatEString(nodeAny.sval.sval);
                 }
                 else if (nodeAny.sval.String && nodeAny.sval.String.sval !== undefined) {
-                    return quote_utils_1.QuoteUtils.escape(nodeAny.sval.String.sval);
+                    return quote_utils_1.QuoteUtils.formatEString(nodeAny.sval.String.sval);
                 }
                 else if (Object.keys(nodeAny.sval).length === 0) {
                     return "''";
                 }
                 else {
-                    return quote_utils_1.QuoteUtils.escape(nodeAny.sval.toString());
+                    return quote_utils_1.QuoteUtils.formatEString(nodeAny.sval.toString());
                 }
             }
             else if (nodeAny.sval === null) {
                 return 'NULL';
             }
             else {
-                return quote_utils_1.QuoteUtils.escape(nodeAny.sval);
+                return quote_utils_1.QuoteUtils.formatEString(nodeAny.sval);
             }
         }
         else if (nodeAny.boolval !== undefined) {
@@ -1835,7 +1838,7 @@ class Deparser {
     }
     String(node, context) {
         if (context.isStringLiteral || context.isEnumValue) {
-            return `'${node.sval || ''}'`;
+            return quote_utils_1.QuoteUtils.formatEString(node.sval || '');
         }
         const value = node.sval || '';
         if (context.parentNodeTypes.includes('DefElem') ||
@@ -5054,6 +5057,18 @@ class Deparser {
                     : argValue;
                 return `${node.defname} = ${quotedValue}`;
             }
+            // Handle CopyStmt WITH clause options - uppercase format without quotes
+            if (context.parentNodeTypes.includes('CopyStmt')) {
+                if (node.defname === 'format' && node.arg && this.getNodeType(node.arg) === 'String') {
+                    const stringData = this.getNodeData(node.arg);
+                    return `FORMAT ${stringData.sval.toUpperCase()}`;
+                }
+                // Handle other COPY options with uppercase defname
+                if (node.arg) {
+                    return `${node.defname.toUpperCase()} ${argValue}`;
+                }
+                return node.defname.toUpperCase();
+            }
             // Handle CREATE OPERATOR and CREATE TYPE context
             if (context.parentNodeTypes.includes('DefineStmt')) {
                 const preservedName = this.preserveOperatorDefElemCase(node.defname);
@@ -5744,8 +5759,7 @@ class Deparser {
             output.push('NULL');
         }
         else if (node.comment) {
-            const escapedComment = node.comment.replace(/'/g, "''");
-            output.push(`'${escapedComment}'`);
+            output.push(quote_utils_1.QuoteUtils.formatEString(node.comment));
         }
         return output.join(' ');
     }

package/esm/deparser.js

@@ -864,13 +864,16 @@ export class Deparser {
             formatStr = '(%s)';
         }
         const boolContext = { ...context, bool: true };
+        // explanation of our syntax/fix below:
+        // return formatStr.replace('%s', andArgs); // ❌ Interprets $ as special syntax
+        // return formatStr.replace('%s', () => andArgs); // ✅ Function callback prevents interpretation
         switch (boolop) {
             case 'AND_EXPR':
                 const andArgs = args.map(arg => this.visit(arg, boolContext)).join(' AND ');
-                return formatStr.replace('%s', andArgs);
+                return formatStr.replace('%s', () => andArgs);
             case 'OR_EXPR':
                 const orArgs = args.map(arg => this.visit(arg, boolContext)).join(' OR ');
-                return formatStr.replace('%s', orArgs);
+                return formatStr.replace('%s', () => orArgs);
             case 'NOT_EXPR':
                 return `NOT (${this.visit(args[0], context)})`;
             default:
@@ -1129,23 +1132,23 @@ export class Deparser {
         else if (nodeAny.sval !== undefined) {
             if (typeof nodeAny.sval === 'object' && nodeAny.sval !== null) {
                 if (nodeAny.sval.sval !== undefined) {
-                    return QuoteUtils.escape(nodeAny.sval.sval);
+                    return QuoteUtils.formatEString(nodeAny.sval.sval);
                 }
                 else if (nodeAny.sval.String && nodeAny.sval.String.sval !== undefined) {
-                    return QuoteUtils.escape(nodeAny.sval.String.sval);
+                    return QuoteUtils.formatEString(nodeAny.sval.String.sval);
                 }
                 else if (Object.keys(nodeAny.sval).length === 0) {
                     return "''";
                 }
                 else {
-                    return QuoteUtils.escape(nodeAny.sval.toString());
+                    return QuoteUtils.formatEString(nodeAny.sval.toString());
                 }
             }
             else if (nodeAny.sval === null) {
                 return 'NULL';
             }
             else {
-                return QuoteUtils.escape(nodeAny.sval);
+                return QuoteUtils.formatEString(nodeAny.sval);
             }
         }
         else if (nodeAny.boolval !== undefined) {
@@ -1832,7 +1835,7 @@ export class Deparser {
     }
     String(node, context) {
         if (context.isStringLiteral || context.isEnumValue) {
-            return `'${node.sval || ''}'`;
+            return QuoteUtils.formatEString(node.sval || '');
         }
         const value = node.sval || '';
         if (context.parentNodeTypes.includes('DefElem') ||
@@ -5051,6 +5054,18 @@ export class Deparser {
                     : argValue;
                 return `${node.defname} = ${quotedValue}`;
             }
+            // Handle CopyStmt WITH clause options - uppercase format without quotes
+            if (context.parentNodeTypes.includes('CopyStmt')) {
+                if (node.defname === 'format' && node.arg && this.getNodeType(node.arg) === 'String') {
+                    const stringData = this.getNodeData(node.arg);
+                    return `FORMAT ${stringData.sval.toUpperCase()}`;
+                }
+                // Handle other COPY options with uppercase defname
+                if (node.arg) {
+                    return `${node.defname.toUpperCase()} ${argValue}`;
+                }
+                return node.defname.toUpperCase();
+            }
             // Handle CREATE OPERATOR and CREATE TYPE context
             if (context.parentNodeTypes.includes('DefineStmt')) {
                 const preservedName = this.preserveOperatorDefElemCase(node.defname);
@@ -5741,8 +5756,7 @@ export class Deparser {
             output.push('NULL');
         }
         else if (node.comment) {
-            const escapedComment = node.comment.replace(/'/g, "''");
-            output.push(`'${escapedComment}'`);
+            output.push(QuoteUtils.formatEString(node.comment));
         }
         return output.join(' ');
     }

package/esm/utils/quote-utils.js

@@ -48,4 +48,34 @@ export class QuoteUtils {
     static escape(literal) {
         return `'${literal.replace(/'/g, "''")}'`;
     }
+    /**
+     * Escapes a string value for use in E-prefixed string literals
+     * Handles both backslashes and single quotes properly
+     */
+    static escapeEString(value) {
+        return value.replace(/\\/g, '\\\\').replace(/'/g, "''");
+    }
+    /**
+     * Formats a string as an E-prefixed string literal with proper escaping
+     * This wraps the complete E-prefix logic including detection and formatting
+     */
+    static formatEString(value) {
+        const needsEscape = QuoteUtils.needsEscapePrefix(value);
+        if (needsEscape) {
+            const escapedValue = QuoteUtils.escapeEString(value);
+            return `E'${escapedValue}'`;
+        }
+        else {
+            return QuoteUtils.escape(value);
+        }
+    }
+    /**
+     * Determines if a string value needs E-prefix for escaped string literals
+     * Detects backslash escape sequences that require E-prefix in PostgreSQL
+     */
+    static needsEscapePrefix(value) {
+        // Always use E'' if the string contains any backslashes,
+        // unless it's a raw \x... bytea-style literal.
+        return !/^\\x[0-9a-fA-F]+$/i.test(value) && value.includes('\\');
+    }
 }

package/esm/utils/statement-splitter.js

@@ -0,0 +1,125 @@
+import { parse } from 'libpg-query';
+/**
+ * Extracts a single statement from SQL using PostgreSQL's location information.
+ * Handles Unicode properly by using byte positions instead of character positions.
+ */
+export function extractStatement(originalSQL, rawStmt, isFirst = false, options = {}) {
+    let extracted = null;
+    // Convert string to buffer to handle byte positions correctly (for Unicode)
+    const sqlBuffer = Buffer.from(originalSQL, 'utf8');
+    if (rawStmt.stmt_location !== undefined && rawStmt.stmt_len !== undefined) {
+        // Use byte positions as provided by PostgreSQL
+        const startByte = rawStmt.stmt_location;
+        const endByte = rawStmt.stmt_location + rawStmt.stmt_len;
+        // Extract using byte positions and convert back to string
+        const extractedBuffer = sqlBuffer.slice(startByte, endByte);
+        extracted = extractedBuffer.toString('utf8');
+    }
+    else if (rawStmt.stmt_location !== undefined && rawStmt.stmt_len === undefined) {
+        // We have location but no length - extract from location to end of file
+        const extractedBuffer = sqlBuffer.slice(rawStmt.stmt_location);
+        extracted = extractedBuffer.toString('utf8');
+    }
+    else if (isFirst && rawStmt.stmt_len !== undefined) {
+        // For first statement when location is missing but we have length
+        const extractedBuffer = sqlBuffer.slice(0, rawStmt.stmt_len);
+        extracted = extractedBuffer.toString('utf8');
+    }
+    else if (isFirst && rawStmt.stmt_location === undefined && rawStmt.stmt_len === undefined) {
+        // For first statement when both location and length are missing, use entire SQL
+        extracted = originalSQL;
+    }
+    if (extracted && options.stripComments !== false) {
+        // Split into lines to handle leading whitespace and comments properly
+        const lines = extracted.split('\n');
+        let startLineIndex = 0;
+        // Find the first line that contains actual SQL content
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i].trim();
+            // Skip empty lines and comment-only lines
+            if (line === '' || line.startsWith('--')) {
+                continue;
+            }
+            startLineIndex = i;
+            break;
+        }
+        // Reconstruct from the first SQL line, preserving the original indentation of that line
+        if (startLineIndex < lines.length) {
+            const resultLines = lines.slice(startLineIndex);
+            extracted = resultLines.join('\n').trim();
+        }
+    }
+    // Final validation unless skipped
+    if (extracted && !options.skipValidation) {
+        const firstLine = extracted.split('\n')[0].trim();
+        const firstWord = firstLine.split(/\s+/)[0].toUpperCase();
+        // Only check for most obvious malformed patterns at the BEGINNING
+        if (
+        // Check if it starts with truncated patterns (not just contains anywhere)
+        extracted.trim().startsWith('ELECT ') || // Missing S from SELECT
+            extracted.trim().startsWith('REATE ') || // Missing C from CREATE  
+            extracted.trim().startsWith('NSERT ') || // Missing I from INSERT
+            // Completely empty or whitespace only
+            extracted.trim().length === 0) {
+            return null; // Invalid extraction, skip this statement
+        }
+    }
+    return extracted;
+}
+/**
+ * Splits SQL text into individual statements using PostgreSQL's parser.
+ * Handles Unicode characters properly and provides detailed location information.
+ */
+export async function splitStatements(sql, options = {}) {
+    const parseResult = await parse(sql);
+    const statements = [];
+    if (!parseResult.stmts) {
+        return statements;
+    }
+    for (let idx = 0; idx < parseResult.stmts.length; idx++) {
+        const stmt = parseResult.stmts[idx];
+        const extracted = extractStatement(sql, stmt, idx === 0, options);
+        if (extracted) {
+            statements.push({
+                statement: extracted,
+                index: idx,
+                location: stmt.stmt_location,
+                length: stmt.stmt_len
+            });
+        }
+    }
+    return statements;
+}
+/**
+ * Utility to generate statement keys for fixtures
+ */
+export function generateStatementKey(relativePath, statementIndex, extension = 'sql') {
+    return `${relativePath.replace(/\.sql$/, '')}-${statementIndex + 1}.${extension}`;
+}
+/**
+ * Test utility to compare byte vs character extraction for debugging Unicode issues
+ */
+export function debugUnicodeExtraction(sql, rawStmt) {
+    const charLength = sql.length;
+    const byteLength = Buffer.from(sql, 'utf8').length;
+    // Character-based extraction (old way)
+    let characterBased = '';
+    if (rawStmt.stmt_location !== undefined && rawStmt.stmt_len !== undefined) {
+        characterBased = sql.substring(rawStmt.stmt_location, rawStmt.stmt_location + rawStmt.stmt_len);
+    }
+    // Byte-based extraction (new way)
+    let byteBased = '';
+    if (rawStmt.stmt_location !== undefined && rawStmt.stmt_len !== undefined) {
+        const sqlBuffer = Buffer.from(sql, 'utf8');
+        const extractedBuffer = sqlBuffer.slice(rawStmt.stmt_location, rawStmt.stmt_location + rawStmt.stmt_len);
+        byteBased = extractedBuffer.toString('utf8');
+    }
+    return {
+        characterBased,
+        byteBased,
+        matches: characterBased === byteBased,
+        unicodeChars: byteLength - charLength,
+        byteLength,
+        charLength
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pgsql-deparser",
-  "version": "17.4.1",
+  "version": "17.5.0",
   "author": "Dan Lynch <pyramation@gmail.com>",
   "description": "PostgreSQL AST Deparser",
   "main": "index.js",
@@ -30,6 +30,7 @@
     "fixtures:ast": "ts-node scripts/make-fixtures-ast.ts",
     "fixtures:sql": "ts-node scripts/make-fixtures-sql.ts",
     "fixtures": "ts-node scripts/make-fixtures.ts",
+    "fixtures:upstream-diff": "ts-node scripts/make-upstream-diff.ts",
     "lint": "eslint . --fix",
     "test": "jest",
     "test:watch": "jest --watch"
@@ -48,7 +49,7 @@
     "libpg-query": "17.3.3"
   },
   "dependencies": {
-    "@pgsql/types": "^17.4.1"
+    "@pgsql/types": "^17.4.2"
   },
-  "gitHead": "254b19f220b9bca358dcb64e761c8ffc5ddf8f82"
+  "gitHead": "a066ec74d2840bcf75bc0f975475508180a621fc"
 }

package/utils/quote-utils.d.ts

@@ -2,4 +2,19 @@ export declare class QuoteUtils {
     static needsQuotes(value: string): boolean;
     static quote(value: any): any;
     static escape(literal: string): string;
+    /**
+     * Escapes a string value for use in E-prefixed string literals
+     * Handles both backslashes and single quotes properly
+     */
+    static escapeEString(value: string): string;
+    /**
+     * Formats a string as an E-prefixed string literal with proper escaping
+     * This wraps the complete E-prefix logic including detection and formatting
+     */
+    static formatEString(value: string): string;
+    /**
+     * Determines if a string value needs E-prefix for escaped string literals
+     * Detects backslash escape sequences that require E-prefix in PostgreSQL
+     */
+    static needsEscapePrefix(value: string): boolean;
 }

package/utils/quote-utils.js

@@ -51,5 +51,35 @@ class QuoteUtils {
     static escape(literal) {
         return `'${literal.replace(/'/g, "''")}'`;
     }
+    /**
+     * Escapes a string value for use in E-prefixed string literals
+     * Handles both backslashes and single quotes properly
+     */
+    static escapeEString(value) {
+        return value.replace(/\\/g, '\\\\').replace(/'/g, "''");
+    }
+    /**
+     * Formats a string as an E-prefixed string literal with proper escaping
+     * This wraps the complete E-prefix logic including detection and formatting
+     */
+    static formatEString(value) {
+        const needsEscape = QuoteUtils.needsEscapePrefix(value);
+        if (needsEscape) {
+            const escapedValue = QuoteUtils.escapeEString(value);
+            return `E'${escapedValue}'`;
+        }
+        else {
+            return QuoteUtils.escape(value);
+        }
+    }
+    /**
+     * Determines if a string value needs E-prefix for escaped string literals
+     * Detects backslash escape sequences that require E-prefix in PostgreSQL
+     */
+    static needsEscapePrefix(value) {
+        // Always use E'' if the string contains any backslashes,
+        // unless it's a raw \x... bytea-style literal.
+        return !/^\\x[0-9a-fA-F]+$/i.test(value) && value.includes('\\');
+    }
 }
 exports.QuoteUtils = QuoteUtils;

package/utils/statement-splitter.d.ts

@@ -0,0 +1,38 @@
+import { RawStmt } from '@pgsql/types';
+export interface ExtractedStatement {
+    statement: string;
+    index: number;
+    location?: number;
+    length?: number;
+}
+export interface StatementSplitterOptions {
+    /** Skip validation for malformed statements */
+    skipValidation?: boolean;
+    /** Strip leading comments from extracted statements */
+    stripComments?: boolean;
+}
+/**
+ * Extracts a single statement from SQL using PostgreSQL's location information.
+ * Handles Unicode properly by using byte positions instead of character positions.
+ */
+export declare function extractStatement(originalSQL: string, rawStmt: RawStmt, isFirst?: boolean, options?: StatementSplitterOptions): string | null;
+/**
+ * Splits SQL text into individual statements using PostgreSQL's parser.
+ * Handles Unicode characters properly and provides detailed location information.
+ */
+export declare function splitStatements(sql: string, options?: StatementSplitterOptions): Promise<ExtractedStatement[]>;
+/**
+ * Utility to generate statement keys for fixtures
+ */
+export declare function generateStatementKey(relativePath: string, statementIndex: number, extension?: string): string;
+/**
+ * Test utility to compare byte vs character extraction for debugging Unicode issues
+ */
+export declare function debugUnicodeExtraction(sql: string, rawStmt: RawStmt): {
+    characterBased: string;
+    byteBased: string;
+    matches: boolean;
+    unicodeChars: number;
+    byteLength: number;
+    charLength: number;
+};

package/utils/statement-splitter.js

@@ -0,0 +1,131 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractStatement = extractStatement;
+exports.splitStatements = splitStatements;
+exports.generateStatementKey = generateStatementKey;
+exports.debugUnicodeExtraction = debugUnicodeExtraction;
+const libpg_query_1 = require("libpg-query");
+/**
+ * Extracts a single statement from SQL using PostgreSQL's location information.
+ * Handles Unicode properly by using byte positions instead of character positions.
+ */
+function extractStatement(originalSQL, rawStmt, isFirst = false, options = {}) {
+    let extracted = null;
+    // Convert string to buffer to handle byte positions correctly (for Unicode)
+    const sqlBuffer = Buffer.from(originalSQL, 'utf8');
+    if (rawStmt.stmt_location !== undefined && rawStmt.stmt_len !== undefined) {
+        // Use byte positions as provided by PostgreSQL
+        const startByte = rawStmt.stmt_location;
+        const endByte = rawStmt.stmt_location + rawStmt.stmt_len;
+        // Extract using byte positions and convert back to string
+        const extractedBuffer = sqlBuffer.slice(startByte, endByte);
+        extracted = extractedBuffer.toString('utf8');
+    }
+    else if (rawStmt.stmt_location !== undefined && rawStmt.stmt_len === undefined) {
+        // We have location but no length - extract from location to end of file
+        const extractedBuffer = sqlBuffer.slice(rawStmt.stmt_location);
+        extracted = extractedBuffer.toString('utf8');
+    }
+    else if (isFirst && rawStmt.stmt_len !== undefined) {
+        // For first statement when location is missing but we have length
+        const extractedBuffer = sqlBuffer.slice(0, rawStmt.stmt_len);
+        extracted = extractedBuffer.toString('utf8');
+    }
+    else if (isFirst && rawStmt.stmt_location === undefined && rawStmt.stmt_len === undefined) {
+        // For first statement when both location and length are missing, use entire SQL
+        extracted = originalSQL;
+    }
+    if (extracted && options.stripComments !== false) {
+        // Split into lines to handle leading whitespace and comments properly
+        const lines = extracted.split('\n');
+        let startLineIndex = 0;
+        // Find the first line that contains actual SQL content
+        for (let i = 0; i < lines.length; i++) {
+            const line = lines[i].trim();
+            // Skip empty lines and comment-only lines
+            if (line === '' || line.startsWith('--')) {
+                continue;
+            }
+            startLineIndex = i;
+            break;
+        }
+        // Reconstruct from the first SQL line, preserving the original indentation of that line
+        if (startLineIndex < lines.length) {
+            const resultLines = lines.slice(startLineIndex);
+            extracted = resultLines.join('\n').trim();
+        }
+    }
+    // Final validation unless skipped
+    if (extracted && !options.skipValidation) {
+        const firstLine = extracted.split('\n')[0].trim();
+        const firstWord = firstLine.split(/\s+/)[0].toUpperCase();
+        // Only check for most obvious malformed patterns at the BEGINNING
+        if (
+        // Check if it starts with truncated patterns (not just contains anywhere)
+        extracted.trim().startsWith('ELECT ') || // Missing S from SELECT
+            extracted.trim().startsWith('REATE ') || // Missing C from CREATE  
+            extracted.trim().startsWith('NSERT ') || // Missing I from INSERT
+            // Completely empty or whitespace only
+            extracted.trim().length === 0) {
+            return null; // Invalid extraction, skip this statement
+        }
+    }
+    return extracted;
+}
+/**
+ * Splits SQL text into individual statements using PostgreSQL's parser.
+ * Handles Unicode characters properly and provides detailed location information.
+ */
+async function splitStatements(sql, options = {}) {
+    const parseResult = await (0, libpg_query_1.parse)(sql);
+    const statements = [];
+    if (!parseResult.stmts) {
+        return statements;
+    }
+    for (let idx = 0; idx < parseResult.stmts.length; idx++) {
+        const stmt = parseResult.stmts[idx];
+        const extracted = extractStatement(sql, stmt, idx === 0, options);
+        if (extracted) {
+            statements.push({
+                statement: extracted,
+                index: idx,
+                location: stmt.stmt_location,
+                length: stmt.stmt_len
+            });
+        }
+    }
+    return statements;
+}
+/**
+ * Utility to generate statement keys for fixtures
+ */
+function generateStatementKey(relativePath, statementIndex, extension = 'sql') {
+    return `${relativePath.replace(/\.sql$/, '')}-${statementIndex + 1}.${extension}`;
+}
+/**
+ * Test utility to compare byte vs character extraction for debugging Unicode issues
+ */
+function debugUnicodeExtraction(sql, rawStmt) {
+    const charLength = sql.length;
+    const byteLength = Buffer.from(sql, 'utf8').length;
+    // Character-based extraction (old way)
+    let characterBased = '';
+    if (rawStmt.stmt_location !== undefined && rawStmt.stmt_len !== undefined) {
+        characterBased = sql.substring(rawStmt.stmt_location, rawStmt.stmt_location + rawStmt.stmt_len);
+    }
+    // Byte-based extraction (new way)
+    let byteBased = '';
+    if (rawStmt.stmt_location !== undefined && rawStmt.stmt_len !== undefined) {
+        const sqlBuffer = Buffer.from(sql, 'utf8');
+        const extractedBuffer = sqlBuffer.slice(rawStmt.stmt_location, rawStmt.stmt_location + rawStmt.stmt_len);
+        byteBased = extractedBuffer.toString('utf8');
+    }
+    return {
+        characterBased,
+        byteBased,
+        matches: characterBased === byteBased,
+        unicodeChars: byteLength - charLength,
+        byteLength,
+        charLength
+    };
+}