pipework 0.287.0 → 0.289.0

package/src/core/config/import-config.ts

@@ -9,10 +9,17 @@ import { setConfigBaseDir, clearConfigBaseDir } from './base-dir.js'
 // resolver) goes through here so relative `migrations`/`schema` paths always
 // resolve against the config file's directory, whichever entry path loads the
 // config first in a process.
-export async function importConfig(configPath: string): Promise<Manifold> {
+//
+// `load` replaces the module loader, which is Node's `import()` by default. A test
+// fork passes the vitest module runner's (#1006): the app's modules then evaluate
+// once, in the same runner as the test files' imports, instead of once under Node.
+export type ConfigLoader = (url: string) => Promise<Record<string, unknown>>
+
+export async function importConfig(configPath: string, load?: ConfigLoader): Promise<Manifold> {
   setConfigBaseDir(dirname(configPath))
   try {
-    const mod: Record<string, unknown> = await import(pathToFileURL(configPath).href)
+    const url = pathToFileURL(configPath).href
+    const mod: Record<string, unknown> = load !== undefined ? await load(url) : await import(url)
     const exported: unknown = mod['default'] ?? mod
 
     if (!isManifold(exported)) {

package/src/core/config/schema.ts

@@ -274,7 +274,9 @@ const vitestProfileSchema = z.object({
   isolate: z.boolean().optional().describe(
     'Run each test file in a fresh worker. Pipework defaults to `false` — pipework\'s own ' +
     'per-test database isolation already separates test state, and re-isolating the worker ' +
-    'costs a process spawn per file. Set `true` for a suite that relies on vitest\'s default.',
+    'costs a process spawn per file. Set `true` for a suite that relies on vitest\'s default. ' +
+    'In a project left at `false`, a file whose text calls vi.resetModules, vi.mock, vi.doMock, ' +
+    'vi.unmock or vi.doUnmock runs in a generated `<name>:registry-isolated` sibling.',
   ),
   database: z.boolean().optional(),
   setupFiles: z.array(z.string()).optional(),

package/src/data/migrate/apply.ts

@@ -1,7 +1,7 @@
 import { readFileSync } from 'node:fs'
 import { createHash } from 'node:crypto'
-import postgres from 'postgres'
-import { scanStatements } from './statements.js'
+import postgres, { type Sql } from 'postgres'
+import { scanStatements, stripComments } from './statements.js'
 import { stripPragmaLines } from './pragma.js'
 import { listMigrations } from './history.js'
 
@@ -52,6 +52,13 @@ export interface ApplyOptions {
    * Issue #378.
    */
   onResume?: (tag: string, fromStatement: number, total: number) => void
+  /**
+   * Called when a resumed no-transaction file's first statement is a
+   * CREATE [UNIQUE] INDEX CONCURRENTLY whose index was left INVALID by the cut
+   * build, after it was dropped and before the statement reruns. `index` is
+   * the schema-qualified name. Issue #1003.
+   */
+  onDropInvalidIndex?: (tag: string, statement: number, index: string) => void
   /**
    * Called for each notice or warning a migration file's own statement raises,
    * with the file's tag, the 1-based statement and the file's statement count.
@@ -193,6 +200,15 @@ export async function applyMigrations(options: ApplyOptions): Promise<number> {
         if (from > 0) options.onResume?.(migration.tag, from + 1, migration.statements.length)
 
         for (let i = from; i < migration.statements.length; i++) {
+          // Only the statement the cut interrupted (the first one run on a
+          // resume), and only its own index: a cut CREATE INDEX CONCURRENTLY
+          // leaves the name held by an INVALID index, which the plain form
+          // fails on ("already exists") and IF NOT EXISTS silently keeps.
+          // A valid index of that name is left alone. Issue #1003.
+          if (i === from && from > 0) {
+            const dropped = await dropInvalidIndexOf(client, migration.statements[i]!)
+            if (dropped !== null) options.onDropInvalidIndex?.(migration.tag, i + 1, dropped)
+          }
           await inStatement(migration.tag, i + 1, migration.statements.length, () => client.unsafe(migration.statements[i]!))
           // Recorded AFTER the statement it describes: marking first would let
           // a death between the mark and the statement skip work silently,
@@ -327,6 +343,50 @@ export function resumePoint(
   return Math.min(recorded.done, migration.statements.length)
 }
 
+const IDENT = String.raw`"(?:[^"]|"")+"|[A-Za-z_][A-Za-z0-9_$]*`
+const CREATE_INDEX_CONCURRENTLY_RE = new RegExp(
+  String.raw`^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+CONCURRENTLY\s+(?:IF\s+NOT\s+EXISTS\s+)?(${IDENT})\s+ON\s+(?:ONLY\s+)?((?:${IDENT})(?:\s*\.\s*(?:${IDENT}))?)`,
+  'i',
+)
+
+function unquoteIdent(token: string): string {
+  return token.startsWith('"') ? token.slice(1, -1).replace(/""/g, '"') : token.toLowerCase()
+}
+
+/**
+ * The index name and table a CREATE [UNIQUE] INDEX CONCURRENTLY statement
+ * names, or null for any other statement (or an unnamed index). The table is
+ * kept as written so Postgres resolves it against the search path itself.
+ */
+export function parseCreateIndexConcurrently(statement: string): { index: string; table: string } | null {
+  const match = CREATE_INDEX_CONCURRENTLY_RE.exec(stripComments(statement))
+  if (match === null) return null
+  return { index: unquoteIdent(match[1]!), table: match[2]! }
+}
+
+/**
+ * Drop the INVALID index a cut CREATE INDEX CONCURRENTLY left under the name
+ * this statement is about to create, on the statement's own table. Returns the
+ * schema-qualified name dropped, or null when the statement is not such a
+ * create or the index is absent or valid. Issue #1003.
+ */
+async function dropInvalidIndexOf(client: Sql, statement: string): Promise<string | null> {
+  const target = parseCreateIndexConcurrently(statement)
+  if (target === null) return null
+  const found = await client.unsafe<{ schema: string; name: string }[]>(
+    `SELECT n.nspname AS schema, c.relname AS name
+       FROM pg_index i
+       JOIN pg_class c ON c.oid = i.indexrelid
+       JOIN pg_namespace n ON n.oid = c.relnamespace
+      WHERE c.relname = $1 AND NOT i.indisvalid AND i.indrelid = to_regclass($2)`,
+    [target.index, target.table],
+  )
+  if (found.length === 0) return null
+  const qualified = `${quoteIdent(found[0]!.schema)}.${quoteIdent(found[0]!.name)}`
+  await client.unsafe(`DROP INDEX CONCURRENTLY IF EXISTS ${qualified}`)
+  return qualified
+}
+
 function quoteIdent(name: string): string {
   return `"${name.replace(/"/g, '""')}"`
 }

package/src/data/migrate/run.ts

@@ -24,12 +24,22 @@ export interface MigrateResult {
   safetyCheck: MigrationCheckResult
   /** No-transaction files picked up half applied and finished. Issue #378. */
   resumed: ResumedMigration[]
+  /** INVALID indexes a cut CREATE INDEX CONCURRENTLY left, dropped before the resumed statement reran. Issue #1003. */
+  droppedInvalidIndexes: DroppedInvalidIndex[]
   /** Notices and warnings raised by migration statements, in arrival order. Issue #970. */
   notices: MigrateNotice[]
   /** Trace/audit tables, columns and day partitions this run provisioned. */
   frameworkTables: FrameworkTablesResult
 }
 
+export interface DroppedInvalidIndex {
+  tag: string
+  /** 1-based statement that rebuilt it. */
+  statement: number
+  /** Schema-qualified, quoted. */
+  index: string
+}
+
 export interface ResumedMigration {
   tag: string
   /** 1-based statement the rerun started from. */
@@ -99,12 +109,14 @@ export async function migrateOne(instance: Manifold, dbConfig: ResolvedDatabase,
   }
 
   const resumed: ResumedMigration[] = []
+  const droppedInvalidIndexes: DroppedInvalidIndex[] = []
   const notices: MigrateNotice[] = []
   const migrationsRun = await applyMigrations({
     migrationsFolder: dbConfig.migrations!,
     migrationsTable: dbConfig.migrationTable,
     connectionUrl: dbConfig.url,
     onResume: (tag, fromStatement, totalStatements) => resumed.push({ tag, fromStatement, totalStatements }),
+    onDropInvalidIndex: (tag, statement, index) => droppedInvalidIndexes.push({ tag, statement, index }),
     onNotice: (tag, statement, statements, notice) => {
       const full: MigrateNotice = { tag, statement, statements, ...notice }
       notices.push(full)
@@ -133,6 +145,7 @@ export async function migrateOne(instance: Manifold, dbConfig: ResolvedDatabase,
     extensions: extensionsCreated,
     safetyCheck,
     resumed,
+    droppedInvalidIndexes,
     notices,
     frameworkTables,
   }

package/src/test/auto-setup.ts

@@ -11,7 +11,10 @@ loadEnvFiles()
 
 let _testContext: TestContext | null = null
 
-const instance = await discoverForTestFork()
+// Loaded through this file's own `import()`, which vitest routes through its module runner
+// (setup files always run there), not through Node: the config's app modules must be the
+// instances the test files import (#1006).
+const instance = await discoverForTestFork(url => import(/* @vite-ignore */ url) as Promise<Record<string, unknown>>)
 
 _setTestState(instance)
 

package/src/test/discover-fork.ts

@@ -1,5 +1,6 @@
 import type { Manifold } from '../core/pipework.js'
 import { discoverInstance } from '../core/config/discover.js'
+import type { ConfigLoader } from '../core/config/import-config.js'
 
 /**
  * Load this test fork's manifold and consume `PIPEWORK_CONFIG`.
@@ -9,9 +10,13 @@ import { discoverInstance } from '../core/config/discover.js'
  * migrate`, a CLI under test — inherits it and migrates the wrong package
  * against the wrong manifold. Discovery failing leaves it set: the variable is
  * part of the diagnosis.
+ *
+ * `load` is the test runner's own `import()` (#1006): the config's module graph is then
+ * evaluated by the same module runner as the test files' imports, so an app module
+ * (and its `pipe.define(...)`) runs once whether or not pipework is inlined.
  */
-export async function discoverForTestFork(): Promise<Manifold> {
-  const instance = await discoverInstance()
+export async function discoverForTestFork(load?: ConfigLoader): Promise<Manifold> {
+  const instance = await discoverInstance(undefined, load)
   delete process.env['PIPEWORK_CONFIG']
   return instance
 }

package/src/test/shared-registry-guard.ts

@@ -0,0 +1,41 @@
+import { vi } from 'vitest'
+import { REGISTRY_CALLS } from './shared-registry.js'
+
+// Setup file for projects that share one module registry per worker (`isolate: false`,
+// non-vm pool). It replaces the `vi` calls that change the registry with a throw naming
+// the test file and the fix. The workspace already routes a file whose own text calls
+// them to an isolated sibling project; this catches a call it could not see, such as one
+// reached through a helper (#1001). Idempotent: setup files re-run for every test file.
+// Listed after the project's own setup files, so a mock a setup file registers on
+// purpose still goes through.
+
+const GUARDED = Symbol.for('pipework.sharedRegistryGuard')
+
+function currentFile(): string {
+  const worker = (globalThis as { __vitest_worker__?: { filepath?: string } }).__vitest_worker__
+  return worker?.filepath ?? 'the running test file'
+}
+
+export function refuse(call: string, file: string): Error {
+  return new Error(
+    `[pipework] vi.${call}() is refused in a shared-registry test project (called from ${file}).\n\n` +
+    '  This project runs with isolate: false: every file in a worker shares one module\n' +
+    '  registry, and pipework itself is not re-evaluated by vi.resetModules() or vi.mock().\n' +
+    '  A reset re-runs the app\'s pipe.define(...) calls against the same registry, so the\n' +
+    '  next file in the worker fails with "Duplicate domain definition", and a mock binds\n' +
+    '  the importers of the file after it.\n' +
+    '  The workspace routes a file to an isolated sibling project when its own text calls\n' +
+    '  vi.resetModules, vi.mock, vi.doMock, vi.unmock or vi.doUnmock. This call was not found in\n' +
+    `  the text of ${file}, so it was not routed.\n` +
+    '  Fix: call it directly in the test file that needs it (not through a helper), or set\n' +
+    '  `isolate: true` on this project or profile in pipework.config.ts.\n',
+  )
+}
+
+const target = vi as unknown as Record<PropertyKey, unknown>
+if (target[GUARDED] !== true) {
+  for (const call of REGISTRY_CALLS) {
+    target[call] = (): never => { throw refuse(call, currentFile()) }
+  }
+  target[GUARDED] = true
+}

package/src/test/shared-registry.ts

@@ -0,0 +1,71 @@
+import { readFileSync, globSync } from 'node:fs'
+import { matchesGlob, resolve, sep } from 'node:path'
+
+// A project with `isolate: false` (and a non-vm pool) keeps one module registry per
+// worker. A test file that resets or mocks modules changes that registry for itself
+// and for every file after it: `vi.resetModules()` re-evaluates the app's modules but
+// not pipework (Node's own cache), so each `pipe.define(...)` runs twice against the
+// one registry and the file after it fails at load (#1001). Such files are routed to an
+// isolated sibling project from their own text; the guard setup file refuses the same
+// calls in shared-registry projects for anything the text scan misses.
+
+/** The `vi` calls that change a worker's module registry or the next file's importers. */
+export const REGISTRY_CALLS = ['resetModules', 'mock', 'doMock', 'unmock', 'doUnmock'] as const
+
+// `vi.mock(`, `vi.mock<T>(`, `vitest.doMock (` and `vi?.resetModules(`. Decided from the
+// file's own text, comments and strings included: a miss is caught by the guard, a
+// false hit only costs one isolated worker.
+const REGISTRY_CALL_PATTERN = new RegExp(
+  `\\b(?:vi|vitest)\\s*\\??\\.\\s*(?:${REGISTRY_CALLS.join('|')})\\s*(?:<[^>]*>)?\\s*\\(`,
+)
+
+export function callsRegistryApi(text: string): boolean {
+  return REGISTRY_CALL_PATTERN.test(text)
+}
+
+/** Name of the isolated sibling that takes the registry-changing files of `project`. */
+export function isolatedSiblingName(project: string): string {
+  return `${project}:registry-isolated`
+}
+
+/** Glob-escape a literal path, so `[id].test.ts` matches only itself. */
+export function escapeGlob(path: string): string {
+  return path.replace(/[\\*?[\]{}()!|+@]/g, '\\$&')
+}
+
+/**
+ * The files `include` selects under `base`, minus `exclude`, as sorted `/`-separated
+ * paths relative to `base`. Vitest globs the same way, so this is the file set the
+ * project would run. A base that does not exist selects nothing.
+ */
+export function listProjectFiles(base: string, include: readonly string[], exclude: readonly string[]): string[] {
+  let found: string[]
+  try {
+    found = globSync([...include], {
+      cwd: base,
+      // Prune the two trees nothing is ever tested in, without walking them.
+      exclude: (path) => {
+        const name = path.split(/[\\/]/).pop()
+        return name === 'node_modules' || name === '.git'
+      },
+    })
+  } catch {
+    return []
+  }
+  const files = found
+    .map(file => file.split(sep).join('/'))
+    .filter(file => !exclude.some(pattern => matchesGlob(file, pattern)))
+  return [...new Set(files)].sort()
+}
+
+/** The files of the project whose own text calls a registry-changing `vi` API. */
+export function findRegistryFiles(base: string, include: readonly string[], exclude: readonly string[]): string[] {
+  return listProjectFiles(base, include, exclude).filter(file => {
+    try {
+      return callsRegistryApi(readFileSync(resolve(base, file), 'utf8'))
+    } catch {
+      return false
+    }
+  })
+}
+

package/src/test/vitest-workspace.ts

@@ -1,4 +1,4 @@
-import { resolve, dirname } from 'node:path'
+import { resolve, dirname, isAbsolute } from 'node:path'
 import { configDefaults } from 'vitest/config'
 import type { ViteUserConfig, UserWorkspaceConfig } from 'vitest/config'
 import type { VitestConfig, VitestModuleConfig, VitestProfileConfig, CoverageConfig } from '../core/config/schema.js'
@@ -10,6 +10,7 @@ import { loadEnvFiles } from '../core/config/env.js'
 import { resolveModules } from '../workspace/discover.js'
 import { workspaceSourcePaths } from '../workspace/source-paths.js'
 import { pipework as pipeworkPlugin } from './vitest-plugin.js'
+import { escapeGlob, findRegistryFiles, isolatedSiblingName } from './shared-registry.js'
 
 const DEFAULT_TEST_TIMEOUT = 30_000
 export const DEFAULT_HOOK_TIMEOUT = 60_000
@@ -22,6 +23,9 @@ export const DEFAULT_HOOK_TIMEOUT = 60_000
 const DEFAULT_POOL = 'forks'
 const DEFAULT_ISOLATE = false
 
+// Setup file that refuses registry-changing `vi` calls in a shared-registry project.
+const SHARED_REGISTRY_GUARD = 'pipework/test/shared-registry'
+
 type PoolName = NonNullable<VitestConfig['pool']>
 
 /**
@@ -199,14 +203,12 @@ export function resolveVitestModules(
 
   for (const [modulePath, moduleConfig] of [...included.entries()].sort(([a], [b]) => a.localeCompare(b))) {
     refuseProjectCoverage(`vitest.modules["${modulePath}"].coverage`, moduleConfig)
-    const project = buildProjectConfig(modulePath, moduleConfig, vitestConfig, configDir, workspacePackages)
-    projects.push(project)
+    pushRouted(projects, buildProjectConfig(modulePath, moduleConfig, vitestConfig, configDir, workspacePackages))
 
     if (moduleConfig.profiles !== undefined) {
       for (const [profileName, profileConfig] of Object.entries(moduleConfig.profiles).sort(([a], [b]) => a.localeCompare(b))) {
         refuseProjectCoverage(`vitest.modules["${modulePath}"].profiles["${profileName}"].coverage`, profileConfig)
-        const profileProject = buildProfileConfig(modulePath, profileName, moduleConfig, profileConfig, vitestConfig, configDir, workspacePackages)
-        projects.push(profileProject)
+        pushRouted(projects, buildProfileConfig(modulePath, profileName, moduleConfig, profileConfig, vitestConfig, configDir, workspacePackages))
       }
     }
   }
@@ -411,6 +413,7 @@ interface ProjectBuildInput {
 
 function buildProject(input: ProjectBuildInput): UserWorkspaceConfig {
   const inlineDeps = buildInlineDeps(input)
+  const shared = sharesRegistry(input)
   return {
     ...input.extra,
     plugins: [
@@ -434,7 +437,7 @@ function buildProject(input: ProjectBuildInput): UserWorkspaceConfig {
           inline: inlineDeps,
         },
       },
-      setupFiles: input.setupFiles,
+      setupFiles: shared ? [...input.setupFiles, SHARED_REGISTRY_GUARD] : input.setupFiles,
       // `vitest.pool` / `vitest.isolate` are dedicated fields, so an explicitly
       // configured one wins over `extra.test`; unset, the default above stands and
       // `extra.test` still overrides it as it always did.
@@ -458,19 +461,25 @@ function buildSetupFiles(database: boolean, userSetupFiles: string[] | undefined
   return files
 }
 
-// Pipework is inlined only where the effective isolate is true: a fresh module
-// registry per file needs it. A shared-registry fork already matches Node's single
-// module instance, so inlining buys nothing there. The vm pools isolate per file
-// whatever `isolate` says, so they always inline. Precedence mirrors buildProject's
-// emitted config: dedicated field, then `extra.test`, then the default.
-function buildInlineDeps(input: ProjectBuildInput): string[] {
+// Whether the project's workers share one module registry: the effective isolate is
+// false and the pool is not a vm pool (those isolate per file whatever `isolate` says).
+// Precedence mirrors buildProject's emitted config: dedicated field, then `extra.test`,
+// then the default.
+function sharesRegistry(input: ProjectBuildInput): boolean {
   const extraTest = 'test' in input.extra ? input.extra['test'] as Record<string, unknown> : {}
   const extraIsolate = typeof extraTest['isolate'] === 'boolean' ? extraTest['isolate'] : undefined
   const extraPool = extraTest['pool']
   const isolate = input.isolate ?? extraIsolate ?? DEFAULT_ISOLATE
   const pool = input.pool ?? extraPool ?? DEFAULT_POOL
+  return !isolate && pool !== 'vmThreads' && pool !== 'vmForks'
+}
+
+// Pipework is inlined only where the registry is not shared: a fresh module registry
+// per file needs it. A shared-registry fork already matches Node's single module
+// instance, so inlining buys nothing there.
+function buildInlineDeps(input: ProjectBuildInput): string[] {
   const deps = new Set<string>()
-  if (isolate || pool === 'vmThreads' || pool === 'vmForks') {
+  if (!sharesRegistry(input)) {
     deps.add('pipework')
   }
   for (const dep of input.inlineDeps) {
@@ -479,6 +488,50 @@ function buildInlineDeps(input: ProjectBuildInput): string[] {
   return [...deps]
 }
 
+// Routing for #1001. `isolate: false` projects keep one module registry per worker, and a
+// file that calls vi.resetModules / vi.mock / vi.doMock / vi.unmock / vi.doUnmock breaks
+// the files after it. Per project, the files its `include` selects (under `dir` or the
+// root, minus its merged `exclude`) are read; those whose own text calls one move to a
+// sibling project `<name>:registry-isolated` that is the same project with `isolate: true`
+// and an `include` of exactly those files, and the project itself excludes them. Each
+// project decides for itself from the file's text, so a file two projects select (a
+// module and its profile, or nested module roots) is routed in each, and still runs once
+// per project as before. Files are visited in sorted order, projects in config order.
+function pushRouted(projects: UserWorkspaceConfig[], project: UserWorkspaceConfig): void {
+  const test = project.test
+  const setup = test?.setupFiles
+  if (test === undefined || typeof test.root !== 'string' || !Array.isArray(setup) || !setup.includes(SHARED_REGISTRY_GUARD)) {
+    projects.push(project)
+    return
+  }
+  const root = test.root
+  const name = String(test.name)
+  const dir = typeof test.dir === 'string' ? (isAbsolute(test.dir) ? test.dir : resolve(root, test.dir)) : root
+  const include = test.include ?? [...configDefaults.include]
+  const exclude = test.exclude ?? [...configDefaults.exclude]
+  const routed = findRegistryFiles(dir, include, exclude)
+  if (routed.length === 0) {
+    projects.push(project)
+    return
+  }
+  const escaped = routed.map(escapeGlob)
+  projects.push({
+    ...project,
+    test: { ...test, exclude: [...exclude, ...escaped] },
+  })
+  projects.push({
+    ...project,
+    test: {
+      ...test,
+      name: isolatedSiblingName(name),
+      isolate: true,
+      include: escaped,
+      setupFiles: setup.filter(file => file !== SHARED_REGISTRY_GUARD),
+      server: { deps: { inline: [...new Set(['pipework', ...((test.server?.deps?.inline as string[] | undefined) ?? [])])] } },
+    },
+  })
+}
+
 function closestMatch(input: string, candidates: string[]): string | null {
   if (candidates.length === 0) return null