polen 0.11.0-next.14 → 0.11.0-next.15

package/src/api/builder/ssg/generate.ts

@@ -0,0 +1,224 @@
+import type { Config } from '#api/config/$'
+import { Routes } from '#api/routes/$'
+import { chunk } from '#lib/kit-temp'
+import { debugPolen } from '#singletons/debug'
+import { Worker } from '@effect/platform'
+import { NodeContext, NodeWorker } from '@effect/platform-node'
+import { FileSystem } from '@effect/platform/FileSystem'
+import { Path } from '@wollybeard/kit'
+import consola from 'consola'
+import { Effect, Layer } from 'effect'
+import getPort from 'get-port'
+import { cpus, totalmem } from 'node:os'
+import {
+  GeneratePagesMessage,
+  type GenerateResult,
+  PageMessage,
+  ServerMessage,
+  StartServerMessage,
+} from './worker-messages.js'
+import { createPageSpawner, createServerSpawner } from './worker-spawners.js'
+
+const debug = debugPolen.sub(`api:ssg:generate`)
+
+export const generate = (config: Config.Config): Effect.Effect<void, Error, FileSystem> =>
+  Effect.gen(function*() {
+    // Read routes from the manifest generated during build
+    const manifest = yield* Routes.Manifest.get(config.paths.project.absolute.build.assets.root)
+    const routes = manifest.routes
+
+    const totalRoutes = routes.length
+
+    // Optimize worker count based on system and workload
+    const cpuCount = cpus().length
+    const memoryGB = totalmem() / (1024 ** 3)
+
+    // Scale workers based on both CPU and memory (SSG is memory intensive)
+    const maxWorkersByCPU = Math.max(1, cpuCount - 1)
+    const maxWorkersByMemory = Math.floor(memoryGB / 2) // ~2GB per worker
+    const optimalWorkers = Math.min(maxWorkersByCPU, maxWorkersByMemory, 8)
+
+    // Dynamic batch sizing for better load distribution
+    const minBatchSize = 10
+    const idealBatchesPerWorker = 3 // Better work stealing
+    const batchSize = Math.max(minBatchSize, Math.ceil(totalRoutes / (optimalWorkers * idealBatchesPerWorker)))
+    const batches = chunk(routes, batchSize)
+
+    consola.info(`SSG Build Plan:`)
+    consola.info(`   Routes: ${totalRoutes} pages to generate`)
+    consola.info(`   Servers: ${optimalWorkers} Polen app instances (to handle requests)`)
+    consola.info(`   Clients: ${optimalWorkers} page generators (to capture HTML)`)
+    consola.info(`   Distribution: ${batches.length} batches × ~${batchSize} pages each`)
+    consola.info(`   System: ${cpuCount} CPUs, ${memoryGB.toFixed(1)}GB RAM`)
+
+    debug(`SSG configuration`, {
+      totalRoutes,
+      optimalWorkers,
+      cpuCount,
+      memoryGB,
+      batchSize,
+      totalBatches: batches.length,
+    })
+
+    // Create worker spawner layers
+    const serverPath = config.paths.project.absolute.build.serverEntrypoint
+    const serverRunnerPath = Path.join(
+      config.paths.framework.sourceDir,
+      'api/builder/ssg/server-runner.worker' + config.paths.framework.sourceExtension,
+    )
+
+    const pageGeneratorPath = Path.join(
+      config.paths.framework.sourceDir,
+      'api/builder/ssg/page-generator.worker' + config.paths.framework.sourceExtension,
+    )
+
+    debug(`Creating worker pools with ${optimalWorkers} workers each`)
+
+    // Create and use worker pools within a scoped context
+    yield* Effect.scoped(
+      Effect.gen(function*() {
+        // Create the server pool with its spawner
+        const serverPool = yield* Worker.makePoolSerialized<ServerMessage>({
+          size: optimalWorkers,
+        }).pipe(
+          Effect.provide(Layer.mergeAll(
+            createServerSpawner(serverRunnerPath),
+            NodeWorker.layerManager,
+          )),
+        )
+
+        // Create the page pool with its spawner
+        const pagePool = yield* Worker.makePoolSerialized<PageMessage>({
+          size: optimalWorkers,
+          concurrency: 1, // One batch at a time per worker
+        }).pipe(
+          Effect.provide(Layer.mergeAll(
+            createPageSpawner(pageGeneratorPath),
+            NodeWorker.layerManager,
+          )),
+        )
+
+        const serverPorts: number[] = []
+
+        consola.info(`\nStarting SSG infrastructure:`)
+        consola.info(`   Launching ${optimalWorkers} Polen app instances...`)
+        debug(`Finding available ports for servers`)
+
+        // Get available ports
+        for (let i = 0; i < optimalWorkers; i++) {
+          const port = yield* Effect.tryPromise({
+            try: () => getPort(),
+            catch: (error) => new Error(`Failed to get port: ${error}`),
+          })
+          serverPorts.push(port)
+        }
+        debug(`Using ports:`, serverPorts)
+
+        // Start servers using Effect
+        yield* Effect.all(
+          serverPorts.map((port) =>
+            serverPool.executeEffect(
+              new StartServerMessage({
+                serverPath,
+                port,
+              }),
+            )
+          ),
+          { concurrency: 'unbounded' },
+        )
+        consola.success(`   All ${optimalWorkers} app instances ready on ports: ${serverPorts.join(', ')}`)
+        debug(`All servers started successfully`)
+
+        // Prepare page generation configs
+        // Each batch is assigned to a server in round-robin fashion
+        const generateConfigs = batches.map((batch, index) => {
+          const assignedPort = serverPorts[index % serverPorts.length]!
+          debug(`Batch ${index + 1}: ${batch.length} routes → server on port ${assignedPort}`)
+          return {
+            batch,
+            assignedPort,
+            index,
+          }
+        })
+
+        consola.info(`\nRoute distribution:`)
+        consola.info(`   Each client processes ~${Math.ceil(batches.length / optimalWorkers)} batches`)
+        consola.info(`   Routes are distributed round-robin across ${optimalWorkers} servers`)
+
+        // Process batches
+        consola.info(`\nGenerating pages...`)
+        const startTime = Date.now()
+        let completedBatches = 0
+        let totalPagesProcessed = 0
+
+        // Process batches using Effect.forEach
+        const results = yield* Effect.forEach(
+          generateConfigs,
+          ({ batch, assignedPort, index }) =>
+            pagePool
+              .executeEffect(
+                new GeneratePagesMessage({
+                  routes: batch,
+                  serverPort: assignedPort,
+                  outputDir: config.paths.project.absolute.build.root,
+                }),
+              )
+              .pipe(
+                Effect.tap((result) =>
+                  Effect.sync(() => {
+                    completedBatches++
+                    totalPagesProcessed += result.processedCount
+
+                    const elapsed = (Date.now() - startTime) / 1000
+                    const pagesPerSec = elapsed > 0 ? (totalPagesProcessed / elapsed).toFixed(1) : '0'
+                    const progress = Math.floor((completedBatches / batches.length) * 100)
+
+                    process.stdout.write(
+                      `\r   Progress: ${completedBatches}/${batches.length} batches (${progress}%) • ${totalPagesProcessed}/${totalRoutes} pages • ${pagesPerSec} pages/s`,
+                    )
+                  })
+                ),
+                Effect.catchAll((error) => {
+                  consola.error(`\nBatch ${index} failed:`, error)
+                  debug(`Batch ${index} error details`, { error, batch })
+                  return Effect.fail(new Error(`Batch ${index} failed: ${error}`))
+                }),
+              ),
+          {
+            concurrency: optimalWorkers,
+            concurrentFinalizers: true,
+          },
+        )
+
+        const successfulResults = results.filter((r) => r.success)
+
+        // Final stats using successful results
+        const totalTime = ((Date.now() - startTime) / 1000).toFixed(1)
+        const totalMemoryMB = successfulResults.reduce((sum, r) => sum + r.memoryUsed, 0) / (1024 * 1024)
+        const avgTimePerBatch = successfulResults.reduce((sum, r) => sum + r.duration, 0) / successfulResults.length
+          / 1000
+
+        consola.success(`\n\nSSG Complete!`)
+        consola.info(`   Generated: ${totalRoutes} pages in ${totalTime}s`)
+        consola.info(`   Performance: ${(totalRoutes / parseFloat(totalTime)).toFixed(1)} pages/sec`)
+        consola.info(`   Avg batch time: ${avgTimePerBatch.toFixed(1)}s`)
+        consola.info(`   Peak memory: ${totalMemoryMB.toFixed(0)}MB`)
+
+        debug(`SSG generation complete`, {
+          totalTime,
+          avgTimePerBatch,
+          totalMemoryMB,
+          successfulBatches: successfulResults.length,
+        })
+        // Cleanup happens automatically when the scope ends
+        debug(`SSG generation complete, cleaning up resources`)
+      }).pipe(
+        // Provide NodeContext for Path service used in page generator
+        Effect.provide(NodeContext.layer),
+      ),
+    )
+
+    consola.info('\nShutting down...')
+    consola.success('Cleanup complete.')
+    debug(`All resources cleaned up`)
+  })

package/src/api/builder/ssg/page-generator.worker.ts

@@ -0,0 +1,110 @@
+/**
+ * Worker that generates static pages by fetching from servers.
+ * This is executed in a worker thread using Effect Worker API.
+ */
+import { debugPolen } from '#singletons/debug'
+import { Path, WorkerRunner } from '@effect/platform'
+import { NodeContext, NodeRuntime, NodeWorkerRunner } from '@effect/platform-node'
+import { Effect, Layer } from 'effect'
+import { promises as fs } from 'node:fs'
+import { type GenerateResult, PageMessage } from './worker-messages.js'
+
+const debug = debugPolen.sub(`api:ssg:page-generator`)
+
+// ============================================================================
+// Generate Pages Handler
+// ============================================================================
+
+const handlers = {
+  GeneratePages: (
+    { routes, serverPort, outputDir }: { routes: readonly string[]; serverPort: number; outputDir: string },
+  ) =>
+    Effect.gen(function*() {
+      const path = yield* Path.Path
+      const startTime = Date.now()
+      let processedCount = 0
+
+      debug(`Starting batch generation`, {
+        totalRoutes: routes.length,
+        serverPort,
+      })
+
+      // Process each route by fetching from the server
+      for (const route of routes) {
+        yield* Effect.tryPromise({
+          try: async () => {
+            // Fetch the page from the server
+            const url = `http://localhost:${serverPort}${route}`
+            const response = await fetch(url)
+
+            if (!response.ok) {
+              throw new Error(`Failed to fetch ${route}: ${response.status} ${response.statusText}`)
+            }
+
+            const html = await response.text()
+
+            // Determine output file path
+            const outputPath = path.join(
+              outputDir,
+              route === '/' ? 'index.html' : `${route.slice(1)}/index.html`,
+            )
+
+            // Ensure directory exists
+            const dir = path.dirname(outputPath)
+            await fs.mkdir(dir, { recursive: true })
+
+            // Write the HTML file
+            await fs.writeFile(outputPath, html, 'utf-8')
+
+            processedCount++
+
+            // Log progress every 5 routes or on last route
+            if (processedCount % 5 === 0 || processedCount === routes.length) {
+              debug(`Progress`, {
+                processedCount,
+                totalRoutes: routes.length,
+                serverPort,
+              })
+            }
+          },
+          catch: (error) => {
+            debug(`Failed to process route`, { route, error })
+            const message = error instanceof Error ? error.message : String(error)
+            return new Error(`Failed to process route ${route}: ${message}`)
+          },
+        })
+      }
+
+      const result: GenerateResult = {
+        success: true,
+        processedCount,
+        duration: Date.now() - startTime,
+        memoryUsed: process.memoryUsage().heapUsed,
+      }
+
+      debug(`Batch generation complete`, result)
+      return result
+    }).pipe(
+      Effect.catchAll((error) =>
+        Effect.succeed({
+          success: false,
+          processedCount: 0,
+          duration: Date.now() - Date.now(),
+          memoryUsed: process.memoryUsage().heapUsed,
+          error: error instanceof Error ? error.message : String(error),
+        })
+      ),
+    ),
+}
+
+// ============================================================================
+// Worker Runner
+// ============================================================================
+
+// Run the worker
+WorkerRunner.launch(
+  Layer.provide(
+    WorkerRunner.layerSerialized(PageMessage, handlers),
+    Layer.merge(NodeWorkerRunner.layer, NodeContext.layer),
+  ),
+).pipe(NodeRuntime.runMain)

package/src/api/builder/ssg/server-runner.worker.ts

@@ -0,0 +1,158 @@
+/**
+ * Worker that runs a Polen server for SSG.
+ * This is executed in a child process using Effect Worker API.
+ */
+import { debugPolen } from '#singletons/debug'
+import { WorkerRunner } from '@effect/platform'
+import { NodeRuntime, NodeWorkerRunner } from '@effect/platform-node'
+import { Duration, Effect, Layer, Scope } from 'effect'
+import { spawn } from 'node:child_process'
+import type { ChildProcess } from 'node:child_process'
+import { ServerMessage } from './worker-messages.js'
+
+const debug = debugPolen.sub(`api:ssg:server-runner`)
+
+// Store the server process for cleanup
+let serverProcess: ChildProcess | null = null
+
+// ============================================================================
+// Handlers
+// ============================================================================
+
+const handlers = {
+  StartServer: ({ serverPath, port }: { serverPath: string; port: number }) =>
+    Effect.gen(function*() {
+      // If there's already a server running, stop it first
+      if (serverProcess) {
+        serverProcess.kill('SIGTERM')
+        yield* Effect.sleep(Duration.millis(500))
+        if (!serverProcess.killed) {
+          serverProcess.kill('SIGKILL')
+        }
+        serverProcess = null
+      }
+
+      // Start the server process
+      const startServer = Effect.sync((): ChildProcess => {
+        debug(`Starting server with command: node ${serverPath}`)
+
+        serverProcess = spawn('node', [serverPath], {
+          env: {
+            ...process.env,
+            PORT: port.toString(),
+          },
+          stdio: ['ignore', 'pipe', 'pipe'],
+        })
+
+        serverProcess.stdout?.on('data', (data) => {
+          debug(`[Server ${port}] stdout`, data.toString().trim())
+        })
+
+        serverProcess.stderr?.on('data', (data) => {
+          debug(`[Server ${port}] stderr`, data.toString().trim())
+        })
+
+        return serverProcess
+      })
+
+      const proc: ChildProcess = yield* startServer
+
+      // Wait for server to be ready with proper interruption support
+      const waitForReady = Effect.async<void>((resume) => {
+        // Handle process errors
+        const errorHandler = (error: Error) => {
+          resume(Effect.die(new Error(`Server failed to start on port ${port}: ${error.message}`)))
+        }
+
+        // Handle process exit
+        const exitHandler = (code: number | null, signal: NodeJS.Signals | null) => {
+          if (code !== 0 && code !== null) {
+            resume(Effect.die(new Error(`Server exited with code ${code} (signal: ${signal})`)))
+          }
+        }
+
+        proc.on('error', errorHandler)
+        proc.on('exit', exitHandler)
+
+        // Check if server is ready by polling
+        let checkInterval: NodeJS.Timeout | null = null
+        const checkServer = async () => {
+          try {
+            const response = await fetch(`http://localhost:${port}/`)
+            if (response.ok || response.status === 404) {
+              debug(`[Server ${port}] Ready!`)
+              if (checkInterval) clearInterval(checkInterval)
+              proc.removeListener('error', errorHandler)
+              proc.removeListener('exit', exitHandler)
+              resume(Effect.succeed(undefined))
+            }
+          } catch (error) {
+            // Server not ready yet, will retry on next interval
+          }
+        }
+
+        // Start polling after initial delay
+        setTimeout(() => {
+          checkServer() // First check
+          checkInterval = setInterval(checkServer, 100)
+        }, 500)
+
+        // Return cleanup function for interruption
+        return Effect.sync(() => {
+          if (checkInterval) {
+            clearInterval(checkInterval)
+          }
+          proc.removeListener('error', errorHandler)
+          proc.removeListener('exit', exitHandler)
+        })
+      })
+
+      // Apply timeout with proper Effect interruption
+      yield* waitForReady.pipe(
+        Effect.timeout(Duration.seconds(30)),
+        Effect.catchTag('TimeoutException', () =>
+          Effect.die(new Error(`Server on port ${port} failed to start within 30 seconds`))),
+      )
+
+      // Add finalizer to ensure cleanup
+      yield* Effect.addFinalizer(() =>
+        Effect.sync(() => {
+          debug(`Finalizer: Stopping server on port ${port}`)
+          if (serverProcess && !serverProcess.killed) {
+            serverProcess.kill('SIGTERM')
+            // Try to kill forcefully after a brief wait
+            setTimeout(() => {
+              if (serverProcess && !serverProcess.killed) {
+                serverProcess.kill('SIGKILL')
+              }
+              serverProcess = null
+            }, 100)
+          }
+        })
+      )
+    }).pipe(Effect.scoped),
+  StopServer: () =>
+    Effect.gen(function*() {
+      if (serverProcess) {
+        serverProcess.kill('SIGTERM')
+        // Give it time to shut down gracefully
+        yield* Effect.sleep(Duration.millis(500))
+        if (!serverProcess.killed) {
+          serverProcess.kill('SIGKILL')
+        }
+        serverProcess = null
+      }
+    }),
+}
+
+// ============================================================================
+// Worker Runner
+// ============================================================================
+
+// Run the worker
+WorkerRunner.launch(
+  Layer.provide(
+    WorkerRunner.layerSerialized(ServerMessage, handlers),
+    NodeWorkerRunner.layer,
+  ),
+).pipe(NodeRuntime.runMain)

package/src/api/builder/ssg/worker-messages.ts

@@ -0,0 +1,69 @@
+import { S } from '#lib/kit-temp/effect'
+
+// ============================================================================
+// Route Schema
+// ============================================================================
+
+const RouteSchema = S.String
+
+// ============================================================================
+// Generate Result Schema
+// ============================================================================
+
+const GenerateResultSchema = S.Struct({
+  success: S.Boolean,
+  processedCount: S.Number,
+  duration: S.Number,
+  memoryUsed: S.Number,
+  error: S.optional(S.String),
+})
+
+export type GenerateResult = S.Schema.Type<typeof GenerateResultSchema>
+
+// ============================================================================
+// Server Messages
+// ============================================================================
+
+export class StartServerMessage extends S.TaggedRequest<StartServerMessage>()(
+  'StartServer',
+  {
+    failure: S.Never,
+    payload: {
+      serverPath: S.String,
+      port: S.Number,
+    },
+    success: S.Void,
+  },
+) {}
+
+export class StopServerMessage extends S.TaggedRequest<StopServerMessage>()(
+  'StopServer',
+  {
+    failure: S.Never,
+    payload: {},
+    success: S.Void,
+  },
+) {}
+
+export type ServerMessage = StartServerMessage | StopServerMessage
+export const ServerMessage = S.Union(StartServerMessage, StopServerMessage)
+
+// ============================================================================
+// Page Generator Messages
+// ============================================================================
+
+export class GeneratePagesMessage extends S.TaggedRequest<GeneratePagesMessage>()(
+  'GeneratePages',
+  {
+    failure: S.String,
+    payload: {
+      routes: S.Array(RouteSchema),
+      serverPort: S.Number,
+      outputDir: S.String,
+    },
+    success: GenerateResultSchema,
+  },
+) {}
+
+export type PageMessage = GeneratePagesMessage
+export const PageMessage = S.Union(GeneratePagesMessage)

package/src/api/builder/ssg/worker-spawners.ts

@@ -0,0 +1,69 @@
+import { NodeWorker } from '@effect/platform-node'
+import { Path } from '@wollybeard/kit'
+import { Worker as NodeWorkerThread } from 'node:worker_threads'
+
+// ============================================================================
+// Server Spawner
+// ============================================================================
+
+/**
+ * Creates a layer for spawning server processes.
+ * Each server runs in its own worker thread.
+ */
+export const createServerSpawner = (workerPath: string) =>
+  NodeWorker.layer((id: number) => {
+    // For TypeScript files, we need to use tsx import
+    const isTypeScript = workerPath.endsWith('.ts')
+
+    if (isTypeScript) {
+      // Use tsx import for TypeScript (Node 20.6+ / 18.19+)
+      return new NodeWorkerThread(workerPath, {
+        execArgv: ['--import', 'tsx'],
+        env: {
+          ...process.env,
+          WORKER_ID: String(id),
+        },
+      })
+    } else {
+      // For JavaScript, use worker threads directly
+      return new NodeWorkerThread(workerPath, {
+        env: {
+          ...process.env,
+          WORKER_ID: String(id),
+        },
+      })
+    }
+  })
+
+// ============================================================================
+// Page Generator Spawner
+// ============================================================================
+
+/**
+ * Creates a layer for spawning page generator workers.
+ * Each generator runs in a worker thread for better performance.
+ */
+export const createPageSpawner = (workerPath: string) =>
+  NodeWorker.layer((id: number) => {
+    // For TypeScript files, we need to use tsx import
+    const isTypeScript = workerPath.endsWith('.ts')
+
+    if (isTypeScript) {
+      // Use tsx import for TypeScript (Node 20.6+ / 18.19+)
+      return new NodeWorkerThread(workerPath, {
+        execArgv: ['--import', 'tsx'],
+        env: {
+          ...process.env,
+          WORKER_ID: String(id),
+        },
+      })
+    } else {
+      // For JavaScript, use worker threads directly
+      return new NodeWorkerThread(workerPath, {
+        env: {
+          ...process.env,
+          WORKER_ID: String(id),
+        },
+      })
+    }
+  })