sonamu 0.7.11 → 0.7.13

package/src/tasks/workflow-manager.ts

@@ -0,0 +1,330 @@
+import { BackendPostgres, OpenWorkflow, type Worker } from "@sonamu-kit/tasks";
+import type {
+  RunnableWorkflow,
+  SchemaInput,
+  SchemaOutput,
+  StandardSchemaV1,
+  StepApi,
+  WorkflowRunHandle,
+  WorkflowSpec,
+} from "@sonamu-kit/tasks/internal";
+import assert from "assert";
+import type { Knex } from "knex";
+import { schedule as cronSchedule, type ScheduledTask } from "node-cron";
+import type { ZodObject } from "zod";
+import type { Context } from "../api/context";
+import { Sonamu } from "../api/sonamu";
+import { Naite } from "../naite/naite";
+import { createMockSSEFactory } from "../stream/sse";
+import type { Executable } from "../types/types";
+import type { WorkflowMetadata } from "./decorator";
+import { StepWrapper } from "./step-wrapper";
+
+export interface WorkflowOptions {
+  // worker에서 동시 실행할 태스크 수, 기본은 CPU 코어 수 - 1개
+  concurrency?: number;
+
+  // worker에서 사용할 pub/sub 여부, 기본은 true
+  usePubSub?: boolean;
+
+  // pub/sub으로 태스크를 수신했을 때 워크플로우 실행까지 지연 시간, 기본은 500ms
+  listenDelay?: number;
+}
+
+// Workflow 함수의 타입, @sonamu-kit/tasks와 다른 점은 step을 한번 감싼 형태.
+export type WorkflowFunction<Input, Output> = (
+  params: Readonly<{
+    input: Input;
+    step: StepWrapper;
+    version: string | null;
+  }>,
+) => Promise<Output> | Output;
+
+// Workflow를 등록할 때를 위한 타입
+export type WorkflowCreateOptions<
+  Input,
+  Output,
+  TSchema extends StandardSchemaV1 | undefined = undefined,
+> = Omit<
+  WorkflowSpec<SchemaOutput<TSchema, Input>, Output, SchemaInput<TSchema, Input>>,
+  "version"
+> & {
+  id: string;
+  version: string | null;
+  function: WorkflowFunction<SchemaOutput<TSchema, Input>, Output>;
+};
+
+export class WorkflowManager {
+  // backend 인스턴스
+  readonly #backend: BackendPostgres;
+
+  // OpenWorkflow 인스턴스
+  readonly #ow: OpenWorkflow;
+
+  // Worker 인스턴스 (없을 수 있으며, 이 때는 분산된 서버 환경에서 DB에 publish만 한다고 가정함)
+  #worker: Worker | null;
+
+  // 파일 경로 -> 파일에 정의된 워크플로우 메타데이터 목록
+  #workflowsMap: Map<string, WorkflowMetadata[]>;
+
+  // Task 이름 -> Task 정보 및 Input 값, Workflow ID
+  #scheduledTasks: Map<
+    string,
+    {
+      task: ScheduledTask;
+      inputFn: Executable<SchemaInput<unknown, unknown>>;
+      workflowId: string;
+    }
+  >;
+
+  private constructor(backend: BackendPostgres) {
+    this.#backend = backend;
+    this.#ow = new OpenWorkflow({ backend });
+    this.#worker = null;
+    this.#workflowsMap = new Map();
+    this.#scheduledTasks = new Map();
+  }
+
+  /**
+   * 정의된 워크플로우 및 워크플로우에 대한 scheduled tasks를 동기화합니다.
+   */
+  async synchronize(workflowMap: Map<string, WorkflowMetadata[]>) {
+    // 1. 삭제된 파일은 일괄 삭제
+    await Promise.allSettled(
+      Array.from(this.#workflowsMap.entries())
+        .filter(([key]) => !workflowMap.has(key))
+        .flatMap(([_, workflows]) =>
+          workflows.map((workflow) => {
+            return Promise.allSettled([
+              ...workflow.schedules.map((schedule) => this.unscheduleTask(schedule.name)),
+              this.unregisterWorkflow(workflow),
+            ]);
+          }),
+        ),
+    );
+
+    // 2. 새로 추가된 파일은 일괄 등록
+    await Promise.allSettled(
+      Array.from(workflowMap.entries())
+        .filter(([key]) => !this.#workflowsMap.has(key))
+        .flatMap(([_, workflows]) =>
+          workflows.map((workflow) => {
+            this.registerWorkflow({
+              id: workflow.id,
+              name: workflow.name,
+              version: workflow.version ?? null,
+              schema: workflow.schema,
+              function: workflow.fn,
+            });
+
+            return Promise.allSettled(
+              workflow.schedules.map((schedule) => this.scheduleTask(workflow, schedule)),
+            );
+          }),
+        ),
+    );
+
+    // 3. 기존과 다른 것을 diff친 다음, 삭제 후 재등록
+    await Promise.allSettled(
+      Array.from(workflowMap.entries())
+        .filter(([key]) => this.#workflowsMap.has(key))
+        .map<[string, [WorkflowMetadata[], WorkflowMetadata[]]]>(([key, newWorkflows]) => {
+          const previousWorkflows = this.#workflowsMap.get(key);
+          assert(previousWorkflows, "previous workflows not found");
+          return [key, [previousWorkflows, newWorkflows]];
+        })
+        .map(async ([_, [previousWorkflows, newWorkflows]]) => {
+          // 기존 것들을 삭제부터 해야함.
+          await Promise.allSettled(
+            previousWorkflows
+              .filter((prevItem) => !newWorkflows.some((newItem) => newItem.id === prevItem.id))
+              .map((prevItem) => {
+                return Promise.allSettled([
+                  ...prevItem.schedules.map((schedule) => this.unscheduleTask(schedule.name)),
+                  this.unregisterWorkflow(prevItem),
+                ]);
+              }),
+          );
+
+          // 새로 추가된 것들을 등록
+          await Promise.allSettled(
+            newWorkflows
+              .filter(
+                (newItem) => !previousWorkflows.some((prevItem) => prevItem.id === newItem.id),
+              )
+              .map(async (newItem) => {
+                this.registerWorkflow({
+                  id: newItem.id,
+                  name: newItem.name,
+                  version: newItem.version ?? null,
+                  schema: newItem.schema,
+                  function: newItem.fn,
+                });
+
+                return Promise.allSettled(
+                  newItem.schedules.map((schedule) => this.scheduleTask(newItem, schedule)),
+                );
+              }),
+          );
+        }),
+    );
+
+    this.#workflowsMap = workflowMap;
+  }
+
+  // 워크플로우를 실행
+  run<Input, Output, TSchema extends StandardSchemaV1 | undefined = undefined>(
+    options: Omit<WorkflowCreateOptions<Input, Output, TSchema>, "function" | "schema" | "id">,
+    input: SchemaInput<TSchema, Input>,
+  ): Promise<WorkflowRunHandle<Output>> {
+    return this.#ow.runWorkflow(
+      {
+        name: options.name,
+        version: options.version ?? undefined,
+      },
+      input,
+    );
+  }
+
+  // cron task를 등록
+  async scheduleTask(
+    workflow: Pick<WorkflowMetadata, "id" | "name" | "version">,
+    schedule: WorkflowMetadata["schedules"][number],
+  ) {
+    const task = cronSchedule(
+      schedule.expression,
+      (async (
+        { name, version }: Pick<WorkflowMetadata, "name" | "version">,
+        { input }: WorkflowMetadata["schedules"][number],
+      ) => {
+        const inputData = await (typeof input === "function"
+          ? Promise.resolve(input())
+          : Promise.resolve(input));
+
+        return this.run({ name, version: version ?? null }, inputData);
+      }).bind(this, workflow, schedule),
+      {
+        name: schedule.name,
+        timezone: Sonamu.config.api.timezone,
+        noOverlap: false,
+      },
+    );
+
+    this.#scheduledTasks.set(schedule.name, {
+      task,
+      inputFn: schedule.input,
+      workflowId: workflow.id,
+    });
+    await task.start();
+  }
+
+  // cron task를 중지
+  async unscheduleTask(name: string) {
+    const taskItem = this.#scheduledTasks.get(name);
+    if (!taskItem) {
+      console.error("scheduled task not found", name);
+      return;
+    }
+
+    this.#scheduledTasks.delete(name);
+    await taskItem.task.stop();
+    await taskItem.task.destroy();
+  }
+
+  // 워크플로우를 등록, 관련된 스케줄은 별도로 등록해야 함.
+  registerWorkflow<Input, Output, TSchema extends StandardSchemaV1 | undefined = undefined>(
+    options: WorkflowCreateOptions<Input, Output, TSchema>,
+  ): RunnableWorkflow<SchemaOutput<TSchema, Input>, Output, SchemaInput<TSchema, Input>> {
+    const fn = async (
+      params: Readonly<{
+        input: SchemaOutput<TSchema, Input>;
+        step: StepApi;
+        version: string | null;
+      }>,
+    ) => {
+      const baseContext = {
+        request: null,
+        reply: null,
+        headers: {},
+        createSSE: (schema: ZodObject) => createMockSSEFactory(schema),
+        naiteStore: Naite.createStore(),
+        user: null,
+        passport: {
+          login: async () => {},
+          logout: () => {},
+        },
+      } as unknown as Context;
+
+      const contextProvider = Sonamu.config.tasks?.contextProvider;
+      const context: Context = contextProvider
+        ? await Promise.resolve(contextProvider(baseContext))
+        : baseContext;
+
+      const step = new StepWrapper(params.step);
+      return Sonamu.asyncLocalStorage.run({ context }, () =>
+        options.function({ input: params.input, step, version: params.version }),
+      );
+    };
+
+    const workflow = this.#ow.defineWorkflow(
+      {
+        name: options.name,
+        version: options.version ?? undefined,
+        schema: options.schema,
+      },
+      fn,
+    );
+    return workflow;
+  }
+
+  // 워크플로우를 등록 해제, 관련된 스케줄은 별도로 해제해야 함.
+  async unregisterWorkflow(workflow: Pick<WorkflowMetadata, "name" | "version" | "id">) {
+    this.#ow.unregisterWorkflow(workflow.name, workflow.version ?? null);
+  }
+
+  // Worker를 설정 후 시작
+  async setupWorker(options: WorkflowOptions) {
+    this.#worker = this.#ow.newWorker(options);
+    await this.#worker.start();
+  }
+
+  // Worker를 중지
+  async stopWorker() {
+    await this.#worker?.stop();
+  }
+
+  // cron task들을 모두 중지
+  async stopSchedules() {
+    await Promise.allSettled(
+      Array.from(this.#scheduledTasks.values()).map(({ task }) => Promise.resolve(task.stop())),
+    );
+  }
+
+  // cron task들을 모두 정리
+  async destroySchedules() {
+    await Promise.allSettled(
+      Array.from(this.#scheduledTasks.values()).map(({ task }) => Promise.resolve(task.destroy())),
+    );
+    this.#scheduledTasks.clear();
+  }
+
+  async destroy() {
+    await this.stopSchedules();
+    await this.stopWorker();
+    await this.destroySchedules();
+    await this.#backend.stop();
+  }
+
+  [Symbol.asyncDispose]() {
+    return this.destroy();
+  }
+
+  // BackendPostgres에서 처리하는 것들이 있어서 Knex 커넥션이 아니라 설정값을 넣어줘야함.
+  static async create(
+    dbConf: Knex.Config,
+    runMigrations: boolean = true,
+  ): Promise<WorkflowManager> {
+    const backend = await BackendPostgres.connect(dbConf, { runMigrations });
+    return new WorkflowManager(backend);
+  }
+}

package/src/template/implementations/generated.template.ts

@@ -190,6 +190,14 @@ export class Template__generated extends Template {
       .map((prop) => (prop.type === "relation" ? `${prop.name}_id` : prop.name))
       .concat("id");
 
+    /**
+     * hasVector props
+     * - vector 타입인 컬럼
+     */
+    const hasVectorColumns = entity.props
+      .filter((prop) => prop.type === "vector" || prop.type === "vector[]")
+      .map((prop) => prop.name);
+
     /**
      * generated props
      * - generated 속성이 있는 컬럼 (INSERT/UPDATE 시 값 제공 불가)
@@ -202,7 +210,8 @@ export class Template__generated extends Template {
       fulltextColumns.length > 0 ||
       virtualProps.length > 0 ||
       hasDefaultColumns.length > 0 ||
-      generatedColumns.length > 0;
+      generatedColumns.length > 0 ||
+      hasVectorColumns.length > 0;
 
     const lines = [
       `export const ${schemaName} = ${schemaBody};`,
@@ -217,15 +226,20 @@ export class Template__generated extends Template {
               (virtualProps.length > 0
                 ? `readonly __virtual__: readonly [${virtualProps.map((prop) => `"${prop}"`).join(", ")}],`
                 : "") +
+              (hasDefaultColumns.length > 0
+                ? `readonly __hasDefault__: readonly [${hasDefaultColumns
+                    .map((col) => `"${col}"`)
+                    .join(", ")}],`
+                : "") +
               (
-                hasDefaultColumns.length > 0
-                  ? `readonly __hasDefault__: readonly [${hasDefaultColumns
+                generatedColumns.length > 0
+                  ? `readonly __generated__: readonly [${generatedColumns
                       .map((col) => `"${col}"`)
                       .join(", ")}],`
                   : ""
               ) +
-              (generatedColumns.length > 0
-                ? `readonly __generated__: readonly [${generatedColumns
+              (hasVectorColumns.length > 0
+                ? `readonly __vector__: readonly [${hasVectorColumns
                     .map((col) => `"${col}"`)
                     .join(", ")}],`
                 : "")
@@ -265,7 +279,6 @@ export class Template__generated extends Template {
     const filterBody = propNodes
       .map((propNode) => propNodeToZodTypeDef(propNode, importKeys))
       .join("\n");
-
     const schemaBody = `
 z.object({
   num: z.number().int().nonnegative(),

package/src/types/types.ts

@@ -13,7 +13,7 @@ export function zArrayable<T extends z.ZodTypeAny>(shape: T): z.ZodUnion<[T, z.Z
 export type DistributiveOmit<T, K extends keyof any> = T extends any ? Omit<T, K> : never;
 
 /*
-  Model-Defintion
+  Model-Definition
 */
 export type GeneratedColumnType = "STORED" | "VIRTUAL";
 export type GeneratedColumn = {
@@ -227,7 +227,7 @@ export type EntityIndex = {
   type: "index" | "unique" | "hnsw" | "ivfflat";
   columns: EntityIndexColumn[];
   name: string;
-  using?: "btree" | "hash" | "gin" | "gist";
+  using?: "btree" | "hash" | "gin" | "gist" | "pgroonga";
   nullsNotDistinct?: boolean; // unique index only
   /**
    * HNSW (Hierarchical Navigable Small World) 인덱스: 각 노드의 최대 연결 수
@@ -492,6 +492,19 @@ export type SubsetQuery = {
 export const SonamuQueryMode = z.enum(["both", "list", "count"]);
 export type SonamuQueryMode = z.infer<typeof SonamuQueryMode>;
 
+/* Semantic Query */
+export const SonamuSemanticParams = z
+  .object({
+    semanticQuery: z.object({
+      embedding: z.array(z.number()).min(1024).max(1024),
+      threshold: z.number().optional(),
+      method: z.enum(["cosine", "l2", "inner_product"]).optional(),
+      which: z.string(),
+    }),
+  })
+  .partial();
+export type SonamuSemanticParams = z.infer<typeof SonamuSemanticParams>;
+
 /* Knex Migration */
 export type KnexError = {
   code: string;
@@ -557,7 +570,7 @@ export type MigrationIndex = {
   type: "unique" | "index" | "hnsw" | "ivfflat";
   columns: EntityIndexColumn[];
   name: string;
-  using?: "btree" | "hash" | "gin" | "gist";
+  using?: "btree" | "hash" | "gin" | "gist" | "pgroonga";
   nullsNotDistinct?: boolean;
   /** HNSW (Hierarchical Navigable Small World): 각 노드의 최대 연결 수 */
   m?: number;
@@ -1143,7 +1156,7 @@ const EntityIndexSchema = z
     type: z.enum(["index", "unique", "hnsw", "ivfflat"]),
     columns: z.array(EntityIndexColumnSchema),
     name: z.string().min(1).max(63),
-    using: z.enum(["btree", "hash", "gin", "gist"]).optional(),
+    using: z.enum(["btree", "hash", "gin", "gist", "pgroonga"]).optional(),
     nullsNotDistinct: z.boolean().optional(),
     m: z.number().optional(),
     efConstruction: z.number().optional(),
@@ -1334,6 +1347,9 @@ export type ManyToManyBaseSchema<FromIdKey extends string, ToIdKey extends strin
   [K in `${ToIdKey}_id`]: number;
 };
 
+// 객체, 함수, 비동기 함수를 모두 포괄하는 타입
+export type Executable<T> = T | Promise<T> | (() => T) | (() => Promise<T>);
+
 export type SonamuFastifyConfig = {
   contextProvider: (
     defaultContext: Pick<Context, "request" | "reply" | "headers" | "createSSE" | "naiteStore"> &

package/src/ui/ai-api.ts

@@ -0,0 +1,60 @@
+// import { convertToModelMessages, type UIMessage } from "ai";
+// import type { FastifyInstance } from "fastify";
+// import { BadRequestException, type FixtureRecord } from "sonamu";
+// import { aiClient } from "./ai-client";
+
+// export async function setAiApi(server: FastifyInstance) {
+//   await aiClient.init();
+
+//   server.post("/api/ai/fixture/chat", async (request, reply) => {
+//     const { messages, fixtureRecords } = request.body as {
+//       messages: UIMessage[];
+//       fixtureRecords?: FixtureRecord[];
+//     };
+
+//     if (!fixtureRecords || fixtureRecords.length === 0) {
+//       throw new BadRequestException("픽스쳐 레코드가 없습니다. 픽스쳐 조회 후 시도하세요.");
+//     }
+
+//     const result = aiClient.handleFixture(convertToModelMessages(messages), fixtureRecords);
+//     const response = result.toUIMessageStreamResponse();
+
+//     reply.raw.writeHead(response.status, Object.fromEntries(response.headers.entries()));
+
+//     if (response.body) {
+//       const reader = response.body.getReader();
+//       while (true) {
+//         const { done, value } = await reader.read();
+//         if (done) break;
+//         reply.raw.write(value);
+//       }
+//     }
+
+//     reply.raw.end();
+//     return reply;
+//   });
+
+//   // Entity/Enum 생성용 AI Chat Stream
+//   server.post("/api/ai/entity/chat", async (request, reply) => {
+//     const { messages } = request.body as {
+//       messages: UIMessage[];
+//     };
+
+//     const result = aiClient.handleEntity(convertToModelMessages(messages));
+//     const response = result.toUIMessageStreamResponse();
+
+//     reply.raw.writeHead(response.status, Object.fromEntries(response.headers.entries()));
+
+//     if (response.body) {
+//       const reader = response.body.getReader();
+//       while (true) {
+//         const { done, value } = await reader.read();
+//         if (done) break;
+//         reply.raw.write(value);
+//       }
+//     }
+
+//     reply.raw.end();
+//     return reply;
+//   });
+// }