spooder 5.1.9 → 5.1.11

package/README.md

@@ -2666,16 +2666,27 @@ try {
 ```
 
 ### Schema Dependencies
-By default, schema files are executed in the order they are provided by the operating system (generally alphabetically).
+By default, schema files are executed in the order they are provided by the operating system (generally alphabetically). Individual revisions within files are always executed in ascending order.
 
-If you have a schema file that depends on one or more other schema files to be executed before it (for example, using foreign keys), you can specify dependencies.
+If a specific revision depends on one or more other schema files to be executed before it (for example, when adding foreign keys), you can specify dependencies at the revision level.
 
 ```sql
--- [deps] table_b_schema.sql, table_c_schema.sql
--- [1] create table_a
-CREATE ...
+-- [1] create table_a (no dependencies)
+CREATE TABLE table_a (
+    id INTEGER PRIMARY KEY,
+    name TEXT NOT NULL
+);
+
+-- [2] add foreign key to table_b
+-- [deps] table_b_schema.sql
+ALTER TABLE table_a ADD COLUMN table_b_id INTEGER REFERENCES table_b(id);
 ```
 
+When a revision specifies dependencies, all revisions of the dependent schema files will be executed before that specific revision runs. This allows you to create tables independently and then add dependencies in later revisions.
+
+>[!IMPORTANT]
+> Dependencies are specified per-revision, not per-file. A `-- [deps]` line applies only to the revision it appears in.
+
 >[!IMPORTANT]
 > Cyclic or missing dependencies will throw an error.
 

package/bun.lock

@@ -14,7 +14,7 @@
   "packages": {
     "@types/bun": ["@types/bun@1.2.21", "", { "dependencies": { "bun-types": "1.2.21" } }, "sha512-NiDnvEqmbfQ6dmZ3EeUO577s4P5bf4HCTXtI6trMc6f6RzirY5IrF3aIookuSpyslFzrnvv2lmEWv5HyC1X79A=="],
 
-    "@types/node": ["@types/node@24.3.0", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-aPTXCrfwnDLj4VvXrm+UUCQjNEvJgNA8s5F1cvwQU+3KNltTOkBm1j30uNLyqqPNe7gE3KFzImYoZEfLhp4Yow=="],
+    "@types/node": ["@types/node@24.3.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-3vXmQDXy+woz+gnrTvuvNrPzekOi+Ds0ReMxw0LzBiK3a+1k0kQn9f2NWk+lgD4rJehFUmYy2gMhJ2ZI+7YP9g=="],
 
     "@types/react": ["@types/react@19.1.12", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-cMoR+FoAf/Jyq6+Df2/Z41jISvGZZ2eTlnsaJRptmZ76Caldwy1odD4xTr/gNV9VLj0AWgg/nmkevIyUfIIq5w=="],
 
@@ -28,7 +28,7 @@
 
     "generate-function": ["generate-function@2.3.1", "", { "dependencies": { "is-property": "^1.0.2" } }, "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ=="],
 
-    "iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="],
+    "iconv-lite": ["iconv-lite@0.7.0", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ=="],
 
     "is-property": ["is-property@1.0.2", "", {}, "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g=="],
 
@@ -38,7 +38,7 @@
 
     "lru.min": ["lru.min@1.1.2", "", {}, "sha512-Nv9KddBcQSlQopmBHXSsZVY5xsdlZkdH/Iey0BlcBYggMd4two7cZnKOK9vmy3nY0O5RGH99z1PCeTpPqszUYg=="],
 
-    "mysql2": ["mysql2@3.14.3", "", { "dependencies": { "aws-ssl-profiles": "^1.1.1", "denque": "^2.1.0", "generate-function": "^2.3.1", "iconv-lite": "^0.6.3", "long": "^5.2.1", "lru.min": "^1.0.0", "named-placeholders": "^1.1.3", "seq-queue": "^0.0.5", "sqlstring": "^2.3.2" } }, "sha512-fD6MLV8XJ1KiNFIF0bS7Msl8eZyhlTDCDl75ajU5SJtpdx9ZPEACulJcqJWr1Y8OYyxsFc4j3+nflpmhxCU5aQ=="],
+    "mysql2": ["mysql2@3.14.5", "", { "dependencies": { "aws-ssl-profiles": "^1.1.1", "denque": "^2.1.0", "generate-function": "^2.3.1", "iconv-lite": "^0.7.0", "long": "^5.2.1", "lru.min": "^1.0.0", "named-placeholders": "^1.1.3", "seq-queue": "^0.0.5", "sqlstring": "^2.3.2" } }, "sha512-40hDf8LPUsuuJ2hFq+UgOuPwt2IFLIRDvMv6ez9hKbXeYuZPxDDwiJW7KdknvOsQqKznaKczOT1kELgFkhDvFg=="],
 
     "named-placeholders": ["named-placeholders@1.1.3", "", { "dependencies": { "lru-cache": "^7.14.1" } }, "sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w=="],
 

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "spooder",
 	"type": "module",
-	"version": "5.1.9",
+	"version": "5.1.11",
 	"module": "./src/api.ts",
 	"bin": {
 		"spooder": "./src/cli.ts"

package/src/api.ts

@@ -820,7 +820,10 @@ export function http_apply_range(file: BunFile, request: Request): BunFile {
 			if (start_is_nan && end_is_nan)
 				return file;
 			
-			file = file.slice(start_is_nan ? file.size - end : start, end_is_nan || start_is_nan ? undefined : end);
+			file = file.slice(
+				start_is_nan ? file.size - end : start,
+				end_is_nan || start_is_nan ? undefined : end + 1
+			);
 		}
 	}
 	return file;

package/src/api_db.ts

@@ -17,45 +17,87 @@ export function db_serialize_set<T extends string>(set: Set<T> | null): string {
 // endregion
 
 // region schema
-interface DependencyTarget {
+interface RevisionTarget {
+	schema_name: string;
 	file_name: string;
+	revision_id: number;
+	sql: string;
+	comment: string;
 	deps: string[];
 }
 
-function order_schema_dep_tree<T extends DependencyTarget>(deps: T[]): T[] {
+function order_revisions_by_dependency(schemas: any[]): RevisionTarget[] {
+	// First, create a map of schema names to all their pending revisions
+	const schema_revisions_map = new Map<string, RevisionTarget[]>();
+	const all_revisions: RevisionTarget[] = [];
+
+	for (const schema of schemas) {
+		const schema_revisions: RevisionTarget[] = [];
+		for (const rev_id of schema.chunk_keys) {
+			const revision = schema.revisions.get(rev_id);
+			const revision_target: RevisionTarget = {
+				schema_name: schema.name,
+				file_name: schema.file_name,
+				revision_id: rev_id,
+				sql: revision.sql,
+				comment: revision.comment,
+				deps: revision.deps || []
+			};
+			schema_revisions.push(revision_target);
+			all_revisions.push(revision_target);
+		}
+		schema_revisions_map.set(schema.name, schema_revisions);
+	}
+
+	// Now sort revisions using topological sort
 	const visited = new Set<string>();
 	const temp = new Set<string>();
-	const result: T[] = [];
-	const map = new Map(deps.map(d => [d.file_name, d]));
- 
-	function visit(node: T): void {
-		if (temp.has(node.file_name))
-			throw new Error(`Cyclic dependency {${node.file_name}}`);
- 
-		if (visited.has(node.file_name))
+	const result: RevisionTarget[] = [];
+
+	function get_revision_key(rev: RevisionTarget): string {
+		return `${rev.schema_name}:${rev.revision_id}`;
+	}
+
+	function visit(revision: RevisionTarget): void {
+		const rev_key = get_revision_key(revision);
+		
+		if (temp.has(rev_key))
+			throw new Error(`Cyclic dependency detected involving ${rev_key}`);
+
+		if (visited.has(rev_key))
 			return;
- 
-		temp.add(node.file_name);
- 
-		for (const dep of node.deps) {
-			const dep_node = map.get(dep);
-			if (!dep_node)
-				throw new Error(`Missing dependency {${dep}}`);
+
+		temp.add(rev_key);
+
+		// Process dependencies: if this revision depends on a schema, 
+		// all pending revisions of that schema must execute first
+		for (const dep_schema_file of revision.deps) {
+			const dep_schema_name = path.basename(dep_schema_file, '.sql');
+			const dep_revisions = schema_revisions_map.get(dep_schema_name);
+			
+			if (!dep_revisions)
+				throw new Error(`Missing dependency schema {${dep_schema_name}} required by ${rev_key}`);
 			
-			visit(dep_node as T);
+			// Visit all pending revisions of the dependency schema
+			for (const dep_revision of dep_revisions) {
+				visit(dep_revision);
+			}
+		}
+
+		temp.delete(rev_key);
+		visited.add(rev_key);
+		result.push(revision);
+	}
+
+	// Visit all revisions
+	for (const revision of all_revisions) {
+		if (!visited.has(get_revision_key(revision))) {
+			visit(revision);
 		}
- 
-		temp.delete(node.file_name);
-		visited.add(node.file_name);
-		result.push(node);
 	}
- 
-	for (const dep of deps)
-		if (!visited.has(dep.file_name))
-			visit(dep);
- 
+
 	return result;
- }
+}
 
 type Row_DBSchema = { db_schema_table_name: string, db_schema_version: number };
 type SchemaVersionMap = Map<string, number>;
@@ -79,26 +121,26 @@ async function db_load_schema(schema_dir: string, schema_versions: SchemaVersion
 		const schema_path = path.join(schema_file_ent.parentPath, schema_file);
 		const schema = await fs.readFile(schema_path, 'utf8');
 
-		const deps = new Array<string>();
-
 		const revisions = new Map();
 		let current_rev_id = 0;
 		let current_rev = '';
 		let current_rev_comment = '';
+		let current_rev_deps = new Array<string>();
 
 		for (const line of schema.split(/\r?\n/)) {
 			const line_identifier = line.match(/^--\s*\[(\d+|deps)\]/);
 			if (line_identifier !== null) {
 				if (line_identifier[1] === 'deps') {
-					// Line contains schema dependencies, example: -- [deps] schema_b.sql,schema_c.sql
+					// Line contains schema dependencies for current revision, example: -- [deps] schema_b.sql,schema_c.sql
 					const deps_raw = line.substring(line.indexOf(']') + 1);
-					deps.push(...deps_raw.split(',').map(e => e.trim().toLowerCase()));
+					current_rev_deps.push(...deps_raw.split(',').map(e => e.trim().toLowerCase()));
 				} else {
 					// New chunk definition detected, store the current chunk and start a new one.
 					if (current_rev_id > 0) {
-						revisions.set(current_rev_id, { sql: current_rev, comment: current_rev_comment });
+						revisions.set(current_rev_id, { sql: current_rev, comment: current_rev_comment, deps: current_rev_deps });
 						current_rev = '';
 						current_rev_comment = '';
+						current_rev_deps = new Array<string>();
 					}
 
 					const rev_number = parseInt(line_identifier[1]);
@@ -118,28 +160,33 @@ async function db_load_schema(schema_dir: string, schema_versions: SchemaVersion
 
 		// There may be something left in current_chunk once we reach end of the file.
 		if (current_rev_id > 0)
-			revisions.set(current_rev_id, { sql: current_rev, comment: current_rev_comment });
+			revisions.set(current_rev_id, { sql: current_rev, comment: current_rev_comment, deps: current_rev_deps });
 
 		if (revisions.size === 0) {
 			db_log(`{${schema_file}} contains no valid revisions`);
 			continue;
 		}
 
-		if (deps.length > 0)
-			db_log(`{${schema_file}} dependencies: ${log_list(deps)}`);
-
 		const current_schema_version = schema_versions.get(schema_name) ?? 0;
+		const pending_revisions = Array.from(revisions.keys()).filter(chunk_id => chunk_id > current_schema_version).sort((a, b) => a - b);
+		
+		// Log any revision-level dependencies
+		for (const rev_id of pending_revisions) {
+			const revision = revisions.get(rev_id);
+			if (revision.deps.length > 0)
+				db_log(`{${schema_file}} revision [${rev_id}] dependencies: ${log_list(revision.deps)}`);
+		}
+
 		schema_out.push({
 			revisions,
 			file_name: schema_file_lower,
 			name: schema_name,
 			current_version: current_schema_version,
-			deps,
-			chunk_keys: Array.from(revisions.keys()).filter(chunk_id => chunk_id > current_schema_version).sort((a, b) => a - b)
+			chunk_keys: pending_revisions
 		});
 	}
 
-	return order_schema_dep_tree(schema_out);
+	return schema_out;
 }
 // endregion
 
@@ -179,20 +226,28 @@ export async function db_update_schema_mysql(db: mysql_types.Connection, schema_
 	`);
 
 	const schemas = await db_load_schema(schema_dir, schema_versions);
-	for (const schema of schemas) {
-		let newest_schema_version = schema.current_version;
-		for (const rev_id of schema.chunk_keys) {
-			const revision = schema.revisions.get(rev_id);
-			const comment_text = revision.comment ? ` "{${revision.comment}}"` : '';
-			db_log(`applying revision [{${rev_id}}]${comment_text} to {${schema.name}}`);
-
-			await db.query(revision.sql);
-			newest_schema_version = rev_id;
-		}
+	const revisions = order_revisions_by_dependency(schemas);
+	
+	// Track the newest version for each schema
+	const schema_versions_tracking = new Map<string, number>();
+	for (const schema of schemas)
+		schema_versions_tracking.set(schema.name, schema.current_version);
+
+	// Execute revisions in dependency order
+	for (const revision of revisions) {
+		const comment_text = revision.comment ? ` "{${revision.comment}}"` : '';
+		db_log(`applying revision [{${revision.revision_id}}]${comment_text} to {${revision.schema_name}}`);
+
+		await db.query(revision.sql);
+		schema_versions_tracking.set(revision.schema_name, revision.revision_id);
+	}
 
-		if (newest_schema_version > schema.current_version) {
-			db_log(`updated table {${schema.name}} to revision {${newest_schema_version}}`);
-			await update_schema_query.execute([newest_schema_version, schema.name]);
+	// Update schema version tracking for all modified schemas
+	for (const [schema_name, newest_version] of schema_versions_tracking) {
+		const original_version = schemas.find(s => s.name === schema_name)?.current_version || 0;
+		if (newest_version > original_version) {
+			db_log(`updated table {${schema_name}} to revision {${newest_version}}`);
+			await update_schema_query.execute([newest_version, schema_name]);
 		}
 	}
 
@@ -451,20 +506,27 @@ export async function db_update_schema_sqlite(db: Database, schema_dir: string,
 			`);
 	
 			const schemas = await db_load_schema(schema_dir, schema_versions);
-	
-			for (const schema of schemas) {
-				let newest_schema_version = schema.current_version;
-				for (const rev_id of schema.chunk_keys) {
-					const revision = schema.revisions.get(rev_id);
-					const comment_text = revision.comment ? ` "{${revision.comment}}"` : '';
-					db_log(`applying revision [{${rev_id}}]${comment_text} to {${schema.name}}`);
-					db.transaction(() => db.run(revision.sql))();
-					newest_schema_version = rev_id;
-				}
-		
-				if (newest_schema_version > schema.current_version) {
-					db_log(`updated table {${schema.name}} to revision {${newest_schema_version}}`);
-					update_schema_query.run(newest_schema_version, schema.name);
+			const revisions = order_revisions_by_dependency(schemas);
+			
+			// Track the newest version for each schema
+			const schema_versions_tracking = new Map<string, number>();
+			for (const schema of schemas)
+				schema_versions_tracking.set(schema.name, schema.current_version);
+
+			// Execute revisions in dependency order
+			for (const revision of revisions) {
+				const comment_text = revision.comment ? ` "{${revision.comment}}"` : '';
+				db_log(`applying revision [{${revision.revision_id}}]${comment_text} to {${revision.schema_name}}`);
+				db.transaction(() => db.run(revision.sql))();
+				schema_versions_tracking.set(revision.schema_name, revision.revision_id);
+			}
+
+			// Update schema version tracking for all modified schemas
+			for (const [schema_name, newest_version] of schema_versions_tracking) {
+				const original_version = schemas.find(s => s.name === schema_name)?.current_version || 0;
+				if (newest_version > original_version) {
+					db_log(`updated table {${schema_name}} to revision {${newest_version}}`);
+					update_schema_query.run(newest_version, schema_name);
 				}
 			}