supatool 0.6.0 → 0.6.2

package/README.md

@@ -89,9 +89,19 @@ Pull schema from remote DB into local files:
 ```bash
 supatool extract --all -o db/schemas
 # Options:
-# --schema public,agent   Specify schemas
+# --schema public,agent   Specify schemas (explicit list)
+# -e auth,storage         Exclude schemas — targets all others automatically
 # -t "user_*"             Filter tables by pattern
-# --force                 Clear output dir before writing
+# --force                 Delete .sql files for objects removed from DB
+```
+
+Unchanged `.sql` files are never overwritten (content is compared excluding the generated header line). Use `--force` to also clean up `.sql` files whose corresponding DB objects have been dropped.
+
+When you have many schemas and only want to exclude a few, use `-e` without `--schema`:
+
+```bash
+# Extract everything except auth and storage schemas
+supatool extract --all -e auth,storage -o db/schemas
 ```
 
 ### Deploy

package/dist/bin/helptext.js

@@ -25,6 +25,7 @@ Common Options:
   -c, --connection <string>    Connection string (postgresql:// or postgres://)
   -o, --output-dir <path>      Output directory
   --schema <schemas>           Target schemas (comma-separated, default: public)
+  --schema-only                Regenerate only the specified schema files; skip index files
   --config <path>              Configuration file path
   -f, --force                  Force overwrite
 

package/dist/bin/supatool.js

@@ -76,7 +76,8 @@ program
     .option('--no-separate', 'Output all objects in same directory')
     .option('--schema <schemas>', 'Target schemas, comma-separated (default: public)')
     .option('--all-schemas', 'Target all schemas in the DB (use with -e to exclude some)')
-    .option('-e, --exclude-schema <schemas>', 'Schemas to exclude, comma-separated (use with --all-schemas)')
+    .option('-e, --exclude-schema <schemas>', 'Schemas to exclude, comma-separated. Without --schema, targets all schemas automatically.')
+    .option('--schema-only', 'Regenerate only the specified schema files; do not touch other schemas or index files (llms.txt, schema_index.json, etc.)')
     .option('--config <path>', 'Configuration file path')
     .option('-f, --force', 'Force overwrite without confirmation')
     .action(async (options) => {
@@ -86,6 +87,18 @@ program
     if (!config.connectionString)
         connectionRequiredError();
     try {
+        if (options.schemaOnly && !options.schema) {
+            console.error('⚠️  --schema-only requires --schema to be specified.');
+            process.exit(1);
+        }
+        if (options.schemaOnly && options.all) {
+            console.error('⚠️  --schema-only cannot be combined with --all. Use --schema-only without --all to regenerate only the specified schema files.');
+            process.exit(1);
+        }
+        if (options.schemaOnly && options.allSchemas) {
+            console.error('⚠️  --schema-only cannot be combined with --all-schemas.');
+            process.exit(1);
+        }
         let schemas = ['public'];
         if (options.schema) {
             schemas = options.schema.split(',').map((s) => s.trim());
@@ -105,6 +118,8 @@ program
             schemas: schemas,
             allSchemas: options.allSchemas || false,
             excludeSchemas,
+            schemasExplicit: !!options.schema,
+            schemaOnly: options.schemaOnly || false,
             version: package_json_1.version
         });
     }

package/dist/sync/definitionExtractor.js

@@ -257,12 +257,11 @@ async function fetchRelationList(client, schemas = ['public']) {
  */
 async function fetchAllSchemas(client) {
     const result = await client.query(`
-    SELECT schema_name
-    FROM information_schema.schemata
-    WHERE schema_name NOT LIKE 'pg_%'
-      AND schema_name != 'information_schema'
-      AND schema_name != 'pg_catalog'
-    ORDER BY schema_name
+    SELECT nspname AS schema_name
+    FROM pg_catalog.pg_namespace
+    WHERE nspname NOT LIKE 'pg_%'
+      AND nspname != 'information_schema'
+    ORDER BY nspname
   `);
     return result.rows.map((r) => r.schema_name);
 }
@@ -1039,7 +1038,7 @@ async function generateCreateTableDDL(client, tableName, schemaName = 'public')
 /**
  * Save definitions to files (merge RLS/triggers into table/view; schema folders when multi-schema)
  */
-async function saveDefinitionsByType(definitions, outputDir, separateDirectories = true, schemas = ['public'], relations = [], rpcTables = [], allSchemas = [], version, tableRlsStatus = []) {
+async function saveDefinitionsByType(definitions, outputDir, separateDirectories = true, schemas = ['public'], relations = [], rpcTables = [], allSchemas = [], version, tableRlsStatus = [], force = false, schemaOnly = false) {
     const fs = await Promise.resolve().then(() => __importStar(require('fs')));
     const path = await Promise.resolve().then(() => __importStar(require('path')));
     const outputDate = new Date().toLocaleDateString('en-CA', { year: 'numeric', month: '2-digit', day: '2-digit' });
@@ -1116,7 +1115,10 @@ async function saveDefinitionsByType(definitions, outputDir, separateDirectories
         ...cronJobs,
         ...customTypes
     ];
-    const multiSchema = schemas.length > 1;
+    // schemaOnly always uses schema subdirectories (outputDir/<schema>/type/)
+    // so that single-schema --schema-only runs write to the correct location
+    // and don't interfere with other schemas already on disk.
+    const multiSchema = schemas.length > 1 || schemaOnly;
     const typeDirNames = {
         table: 'tables',
         view: 'views',
@@ -1128,6 +1130,7 @@ async function saveDefinitionsByType(definitions, outputDir, separateDirectories
         fs.mkdirSync(outputDir, { recursive: true });
     }
     const fsPromises = await Promise.resolve().then(() => __importStar(require('fs/promises')));
+    const writtenPaths = new Set();
     for (const def of toWrite) {
         const typeDir = typeDirNames[def.type];
         const baseTypeDir = separateDirectories ? typeDir : '.';
@@ -1140,10 +1143,67 @@ async function saveDefinitionsByType(definitions, outputDir, separateDirectories
         const fileName = `${def.name}.sql`;
         const filePath = path.join(targetDir, fileName);
         const ddlWithNewline = def.ddl.endsWith('\n') ? def.ddl : def.ddl + '\n';
-        await fsPromises.writeFile(filePath, headerComment + ddlWithNewline);
+        const newContent = headerComment + ddlWithNewline;
+        writtenPaths.add(filePath);
+        // Skip write if content unchanged (ignore header line which contains the date)
+        if (fs.existsSync(filePath)) {
+            const existingContent = await fsPromises.readFile(filePath, 'utf8');
+            const stripHeader = (c) => c.split('\n').slice(1).join('\n');
+            if (stripHeader(existingContent) === stripHeader(newContent)) {
+                continue;
+            }
+        }
+        await fsPromises.writeFile(filePath, newContent);
+    }
+    // When force: delete SQL files that no longer correspond to any extracted object.
+    // With schemaOnly: restrict deletion to the target schema directories only.
+    if (force && fs.existsSync(outputDir)) {
+        const deleteStaleSqlFiles = (dir) => {
+            for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+                const fullPath = path.join(dir, entry.name);
+                if (entry.isDirectory()) {
+                    deleteStaleSqlFiles(fullPath);
+                }
+                else if (entry.name.endsWith('.sql') && !writtenPaths.has(fullPath)) {
+                    fs.unlinkSync(fullPath);
+                }
+            }
+        };
+        if (schemaOnly) {
+            // Only scan within the target schema directories
+            for (const schemaName of schemas) {
+                const schemaDir = path.join(outputDir, schemaName);
+                if (fs.existsSync(schemaDir)) {
+                    deleteStaleSqlFiles(schemaDir);
+                }
+            }
+        }
+        else {
+            deleteStaleSqlFiles(outputDir);
+        }
+    }
+    // When schemaOnly: skip index file regeneration to avoid overwriting
+    // shared index files (llms.txt, schema_index.json, etc.) with partial data.
+    if (schemaOnly) {
+        return;
     }
     await generateIndexFile(toWrite, outputDir, separateDirectories, multiSchema, relations, rpcTables, allSchemas, schemas, version, tableRlsStatus);
 }
+/**
+ * Write a file only when content has changed, to avoid unnecessary Git diffs.
+ * When skipFirstLine is true, the first line (typically a date-bearing header) is
+ * excluded from the comparison so that daily date changes don't trigger writes.
+ */
+function writeFileIfChanged(filePath, newContent, skipFirstLine = false) {
+    const fs = require('fs');
+    if (fs.existsSync(filePath)) {
+        const existing = fs.readFileSync(filePath, 'utf8');
+        const strip = (c) => skipFirstLine ? c.split('\n').slice(1).join('\n') : c;
+        if (strip(existing) === strip(newContent))
+            return;
+    }
+    fs.writeFileSync(filePath, newContent, 'utf8');
+}
 /**
  * Generate index file for DB objects (RLS/triggers already merged into table/view)
  */
@@ -1265,8 +1325,8 @@ async function generateIndexFile(definitions, outputDir, separateDirectories = t
     }
     const readmePath = path.join(outputDir, 'README.md');
     const llmsPath = path.join(outputDir, 'llms.txt');
-    fs.writeFileSync(readmePath, readmeContent);
-    fs.writeFileSync(llmsPath, llmsContent);
+    writeFileIfChanged(readmePath, readmeContent, true);
+    writeFileIfChanged(llmsPath, llmsContent, true);
     // schema_index.json (same data for agents that parse JSON)
     const schemaIndex = {
         objects: definitions.map(def => {
@@ -1293,7 +1353,7 @@ async function generateIndexFile(definitions, outputDir, separateDirectories = t
             }
             : undefined
     };
-    fs.writeFileSync(path.join(outputDir, 'schema_index.json'), JSON.stringify(schemaIndex, null, 2), 'utf8');
+    writeFileIfChanged(path.join(outputDir, 'schema_index.json'), JSON.stringify(schemaIndex, null, 2) + '\n');
     // schema_summary.md (one-file overview for AI) — include RLS status per table
     let summaryMd = '# Schema summary\n\n';
     const tableDefs = definitions.filter(d => d.type === 'table' || d.type === 'view');
@@ -1326,7 +1386,7 @@ async function generateIndexFile(definitions, outputDir, separateDirectories = t
         summaryMd += `- Extracted: ${allSchemas.filter(s => extractedSet.has(s)).join(', ') || '(none)'}\n`;
         summaryMd += `- Not extracted: ${allSchemas.filter(s => !extractedSet.has(s)).join(', ') || '(none)'}\n`;
     }
-    fs.writeFileSync(path.join(outputDir, 'schema_summary.md'), summaryMd, 'utf8');
+    writeFileIfChanged(path.join(outputDir, 'schema_summary.md'), summaryMd);
     // RLS disabled tables warning doc (tables only; RLS enabled with 0 policies is not warned)
     const rlsNotEnabled = tableRlsStatus.filter(s => !s.rlsEnabled);
     if (rlsNotEnabled.length > 0) {
@@ -1337,14 +1397,14 @@ async function generateIndexFile(definitions, outputDir, separateDirectories = t
         rlsNotEnabled.forEach(s => {
             warnMd += `| ${s.schema} | ${s.table} |\n`;
         });
-        fs.writeFileSync(path.join(outputDir, 'rls_warnings.md'), warnMd, 'utf8');
+        writeFileIfChanged(path.join(outputDir, 'rls_warnings.md'), warnMd);
     }
 }
 /**
  * Classify and output definitions
  */
 async function extractDefinitions(options) {
-    const { connectionString, outputDir, separateDirectories = true, tablesOnly = false, viewsOnly = false, all = false, tablePattern = '*', force = false, schemas: schemasOption = ['public'], excludeSchemas = [], allSchemas: useAllSchemas = false, version } = options;
+    const { connectionString, outputDir, separateDirectories = true, tablesOnly = false, viewsOnly = false, all = false, tablePattern = '*', force = false, schemas: schemasOption = ['public'], excludeSchemas = [], allSchemas: useAllSchemas = false, schemasExplicit = false, schemaOnly = false, version } = options;
     // schemas will be resolved after DB connect when useAllSchemas is true
     let schemas = schemasOption;
     // Disable Node.js SSL certificate verification
@@ -1364,18 +1424,13 @@ async function extractDefinitions(options) {
     }
     // URL encode password part
     let encodedConnectionString = connectionString;
-    console.log('🔍 Original connection string:', connectionString);
     try {
         // Special handling when password contains @
         if (connectionString.includes('@') && connectionString.split('@').length > 2) {
-            console.log('⚠️ Password contains @, executing special handling');
             // Use last @ as delimiter
             const parts = connectionString.split('@');
             const lastPart = parts.pop(); // Last part (host:port/database)
             const firstParts = parts.join('@'); // First part (postgresql://user:password)
-            console.log('   Split result:');
-            console.log('   First part:', firstParts);
-            console.log('   Last part:', lastPart);
             // Encode password part
             const colonIndex = firstParts.lastIndexOf(':');
             if (colonIndex > 0) {
@@ -1383,50 +1438,40 @@ async function extractDefinitions(options) {
                 const password = firstParts.substring(colonIndex + 1);
                 const encodedPassword = encodeURIComponent(password);
                 encodedConnectionString = `${protocolAndUser}:${encodedPassword}@${lastPart}`;
-                console.log('   Encode result:');
-                console.log('   Protocol+User:', protocolAndUser);
-                console.log('   Original password:', password);
-                console.log('   Encoded password:', encodedPassword);
-                console.log('   Final connection string:', encodedConnectionString);
             }
         }
         else {
-            console.log('✅ Executing normal URL parsing');
             // Normal URL parsing
             const url = new URL(connectionString);
-            // Handle username containing dots
-            if (url.username && url.username.includes('.')) {
-                console.log(`Username (with dots): ${url.username}`);
-            }
             if (url.password) {
                 // Encode only password part
                 const encodedPassword = encodeURIComponent(url.password);
                 url.password = encodedPassword;
                 encodedConnectionString = url.toString();
-                console.log('   Password encoded:', encodedPassword);
             }
         }
         // Add SSL settings for Supabase connection
         if (!encodedConnectionString.includes('sslmode=')) {
             const separator = encodedConnectionString.includes('?') ? '&' : '?';
             encodedConnectionString += `${separator}sslmode=require`;
-            console.log('   SSL setting added:', encodedConnectionString);
         }
-        // Display debug info (password hidden)
-        const debugUrl = new URL(encodedConnectionString);
-        const maskedPassword = debugUrl.password ? '*'.repeat(debugUrl.password.length) : '';
-        debugUrl.password = maskedPassword;
-        console.log('🔍 Connection info:');
-        console.log(`   Host: ${debugUrl.hostname}`);
-        console.log(`   Port: ${debugUrl.port}`);
-        console.log(`   Database: ${debugUrl.pathname.slice(1)}`);
-        console.log(`   User: ${debugUrl.username}`);
-        console.log(`   SSL: ${debugUrl.searchParams.get('sslmode') || 'require'}`);
+        // Debug-only: connection info (no credentials)
+        if (process.env.SUPATOOL_DEBUG) {
+            const debugUrl = new URL(encodedConnectionString);
+            console.log('🔍 Connection info:');
+            console.log(`   Host: ${debugUrl.hostname}`);
+            console.log(`   Port: ${debugUrl.port}`);
+            console.log(`   Database: ${debugUrl.pathname.slice(1)}`);
+            console.log(`   User: ${debugUrl.username}`);
+            console.log(`   SSL: ${debugUrl.searchParams.get('sslmode') || 'require'}`);
+        }
     }
     catch (error) {
         // Use original string if URL parsing fails
-        console.warn('Failed to parse connection string URL. May contain special characters.');
-        console.warn('Error details:', error instanceof Error ? error.message : String(error));
+        if (process.env.SUPATOOL_DEBUG) {
+            console.warn('Failed to parse connection string URL. May contain special characters.');
+            console.warn('Error details:', error instanceof Error ? error.message : String(error));
+        }
     }
     const fs = await Promise.resolve().then(() => __importStar(require('fs')));
     const readline = await Promise.resolve().then(() => __importStar(require('readline')));
@@ -1459,14 +1504,11 @@ async function extractDefinitions(options) {
         }
     });
     try {
-        // Debug before connect
-        console.log('🔧 Connection settings:');
-        console.log(`   SSL: rejectUnauthorized=false`);
-        console.log(`   Connection string length: ${encodedConnectionString.length}`);
         await client.connect();
         spinner.text = 'Connected to database';
-        // Resolve schemas: when --all-schemas, fetch all from DB and subtract excludeSchemas
-        if (useAllSchemas) {
+        // Resolve schemas: --all-schemas or -e alone (without explicit --schema) → fetch all from DB and subtract excludeSchemas
+        const useAllSchemasEffective = useAllSchemas || (excludeSchemas.length > 0 && !schemasExplicit);
+        if (useAllSchemasEffective) {
             const SYSTEM_SCHEMAS = ['information_schema', 'pg_catalog', 'pg_toast', 'pg_temp_1', 'pg_toast_temp_1'];
             const discovered = await fetchAllSchemas(client);
             schemas = discovered.filter(s => !SYSTEM_SCHEMAS.includes(s) && !excludeSchemas.includes(s));
@@ -1624,13 +1666,9 @@ async function extractDefinitions(options) {
                 console.warn('RLS status fetch skipped:', err);
             }
         }
-        // When force: remove output dir then write (so removed tables don't leave files)
-        if (force && fs.existsSync(outputDir)) {
-            fs.rmSync(outputDir, { recursive: true });
-        }
         // Save definitions (table+RLS+triggers merged, schema folders)
         spinner.text = 'Saving definitions to files...';
-        await saveDefinitionsByType(allDefinitions, outputDir, separateDirectories, schemas, relations, rpcTables, allSchemas, version, tableRlsStatus);
+        await saveDefinitionsByType(allDefinitions, outputDir, separateDirectories, schemas, relations, rpcTables, allSchemas, version, tableRlsStatus, force, schemaOnly);
         // Warn at extract time when any table has RLS disabled
         const rlsNotEnabled = tableRlsStatus.filter(s => !s.rlsEnabled);
         if (rlsNotEnabled.length > 0) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "supatool",
-  "version": "0.6.0",
+  "version": "0.6.2",
   "description": "CLI for PostgreSQL (Cloud SQL / Supabase): extract schema to files, deploy schema diffs, apply migrations, seed export.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -27,6 +27,14 @@
     "database",
     "migration"
   ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/idea-garage/supatool"
+  },
+  "homepage": "https://github.com/idea-garage/supatool#readme",
+  "bugs": {
+    "url": "https://github.com/idea-garage/supatool/issues"
+  },
   "author": "IdeaGarage",
   "license": "MIT",
   "dependencies": {