sonamu 0.7.10 → 0.7.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/config.d.ts +10 -3
- package/dist/api/config.d.ts.map +1 -1
- package/dist/api/config.js +2 -1
- package/dist/api/sonamu.d.ts +4 -0
- package/dist/api/sonamu.d.ts.map +1 -1
- package/dist/api/sonamu.js +36 -2
- package/dist/bin/cli.js +121 -117
- package/dist/database/base-model.d.ts +10 -50
- package/dist/database/base-model.d.ts.map +1 -1
- package/dist/database/base-model.js +19 -84
- package/dist/database/base-model.types.d.ts +4 -4
- package/dist/database/base-model.types.d.ts.map +1 -1
- package/dist/database/base-model.types.js +1 -1
- package/dist/database/db.d.ts +1 -0
- package/dist/database/db.d.ts.map +1 -1
- package/dist/database/db.js +24 -13
- package/dist/database/puri-subset.test-d.js +1 -1
- package/dist/database/puri-subset.types.d.ts +1 -0
- package/dist/database/puri-subset.types.d.ts.map +1 -1
- package/dist/database/puri-subset.types.js +2 -2
- package/dist/database/puri.d.ts +96 -1
- package/dist/database/puri.d.ts.map +1 -1
- package/dist/database/puri.js +214 -2
- package/dist/database/puri.types.d.ts +60 -5
- package/dist/database/puri.types.d.ts.map +1 -1
- package/dist/database/puri.types.js +2 -3
- package/dist/database/puri.types.test-d.js +1 -1
- package/dist/database/upsert-builder.d.ts +3 -1
- package/dist/database/upsert-builder.d.ts.map +1 -1
- package/dist/database/upsert-builder.js +19 -4
- package/dist/entity/entity-manager.d.ts +5 -4
- package/dist/entity/entity-manager.d.ts.map +1 -1
- package/dist/entity/entity-manager.js +8 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -3
- package/dist/migration/code-generation.d.ts.map +1 -1
- package/dist/migration/code-generation.js +33 -2
- package/dist/migration/postgresql-schema-reader.d.ts.map +1 -1
- package/dist/migration/postgresql-schema-reader.js +53 -22
- package/dist/naite/messaging-types.d.ts.map +1 -1
- package/dist/naite/messaging-types.js +1 -1
- package/dist/naite/naite-reporter.d.ts +7 -4
- package/dist/naite/naite-reporter.d.ts.map +1 -1
- package/dist/naite/naite-reporter.js +45 -21
- package/dist/naite/naite.js +2 -2
- package/dist/stream/sse.d.ts +2 -6
- package/dist/stream/sse.d.ts.map +1 -1
- package/dist/stream/sse.js +9 -3
- package/dist/syncer/api-parser.js +5 -1
- package/dist/syncer/file-patterns.d.ts +1 -1
- package/dist/syncer/file-patterns.d.ts.map +1 -1
- package/dist/syncer/file-patterns.js +6 -5
- package/dist/syncer/module-loader.d.ts +5 -0
- package/dist/syncer/module-loader.d.ts.map +1 -1
- package/dist/syncer/module-loader.js +17 -1
- package/dist/syncer/syncer.d.ts +3 -0
- package/dist/syncer/syncer.d.ts.map +1 -1
- package/dist/syncer/syncer.js +12 -2
- package/dist/tasks/decorator.d.ts +26 -0
- package/dist/tasks/decorator.d.ts.map +1 -0
- package/dist/tasks/decorator.js +28 -0
- package/dist/tasks/step-wrapper.d.ts +18 -0
- package/dist/tasks/step-wrapper.d.ts.map +1 -0
- package/dist/tasks/step-wrapper.js +38 -0
- package/dist/tasks/workflow-manager.d.ts +40 -0
- package/dist/tasks/workflow-manager.d.ts.map +1 -0
- package/dist/tasks/workflow-manager.js +193 -0
- package/dist/template/implementations/generated.template.d.ts.map +1 -1
- package/dist/template/implementations/generated.template.js +7 -3
- package/dist/template/implementations/model.template.js +2 -2
- package/dist/template/zod-converter.d.ts.map +1 -1
- package/dist/template/zod-converter.js +4 -2
- package/dist/types/types.d.ts +28 -11
- package/dist/types/types.d.ts.map +1 -1
- package/dist/types/types.js +18 -2
- package/dist/utils/console-util.js +2 -2
- package/dist/utils/formatter.d.ts.map +1 -1
- package/dist/utils/formatter.js +10 -2
- package/dist/utils/model.d.ts +9 -2
- package/dist/utils/model.d.ts.map +1 -1
- package/dist/utils/model.js +16 -1
- package/dist/utils/type-utils.d.ts.map +1 -1
- package/dist/utils/type-utils.js +3 -1
- package/dist/vector/embedding.d.ts +2 -5
- package/dist/vector/embedding.d.ts.map +1 -1
- package/dist/vector/embedding.js +3 -7
- package/dist/vector/types.d.ts.map +1 -1
- package/dist/vector/types.js +1 -1
- package/package.json +4 -2
- package/src/api/config.ts +15 -8
- package/src/api/sonamu.ts +43 -2
- package/src/bin/cli.ts +58 -54
- package/src/database/base-model.ts +21 -128
- package/src/database/base-model.types.ts +3 -4
- package/src/database/db.ts +28 -18
- package/src/database/puri-subset.test-d.ts +1 -0
- package/src/database/puri-subset.types.ts +2 -0
- package/src/database/puri.ts +292 -1
- package/src/database/puri.types.test-d.ts +1 -1
- package/src/database/puri.types.ts +81 -7
- package/src/database/upsert-builder.ts +27 -9
- package/src/entity/entity-manager.ts +9 -0
- package/src/index.ts +1 -1
- package/src/migration/code-generation.ts +40 -1
- package/src/migration/postgresql-schema-reader.ts +53 -22
- package/src/naite/messaging-types.ts +43 -44
- package/src/naite/naite-reporter.ts +51 -20
- package/src/naite/naite.ts +1 -1
- package/src/shared/app.shared.ts.txt +13 -0
- package/src/shared/web.shared.ts.txt +13 -0
- package/src/stream/sse.ts +15 -3
- package/src/syncer/api-parser.ts +4 -0
- package/src/syncer/file-patterns.ts +11 -9
- package/src/syncer/module-loader.ts +35 -0
- package/src/syncer/syncer.ts +14 -0
- package/src/tasks/decorator.ts +71 -0
- package/src/tasks/step-wrapper.ts +84 -0
- package/src/tasks/workflow-manager.ts +330 -0
- package/src/template/implementations/generated.template.ts +19 -6
- package/src/template/implementations/model.template.ts +1 -1
- package/src/template/zod-converter.ts +3 -0
- package/src/types/types.ts +23 -4
- package/src/utils/console-util.ts +1 -1
- package/src/utils/formatter.ts +8 -1
- package/src/utils/model.ts +26 -2
- package/src/utils/type-utils.ts +2 -0
- package/src/vector/embedding.ts +2 -8
- package/src/vector/types.ts +1 -2
- package/dist/vector/vector-search.d.ts +0 -47
- package/dist/vector/vector-search.d.ts.map +0 -1
- package/dist/vector/vector-search.js +0 -176
- package/src/vector/vector-search.ts +0 -261
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import type {
|
|
3
|
+
SchemaInput,
|
|
4
|
+
SchemaOutput,
|
|
5
|
+
StandardSchemaV1,
|
|
6
|
+
WorkflowSpec,
|
|
7
|
+
} from "@sonamu-kit/tasks/internal";
|
|
8
|
+
import inflection from "inflection";
|
|
9
|
+
import type { Executable } from "../types/types";
|
|
10
|
+
import type { WorkflowFunction } from "./workflow-manager";
|
|
11
|
+
|
|
12
|
+
// 워크플로우의 메타데이터 객체
|
|
13
|
+
export interface WorkflowMetadata {
|
|
14
|
+
type: "workflow";
|
|
15
|
+
id: string;
|
|
16
|
+
name: string;
|
|
17
|
+
version: string | null;
|
|
18
|
+
schema: StandardSchemaV1<unknown, unknown> | undefined;
|
|
19
|
+
schedules: {
|
|
20
|
+
name: string;
|
|
21
|
+
expression: string;
|
|
22
|
+
input: Executable<SchemaInput<unknown, unknown> | undefined>;
|
|
23
|
+
}[];
|
|
24
|
+
fn: WorkflowFunction<unknown, unknown>;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// 워크플로우 정의 과정에서의 옵션
|
|
28
|
+
export type DefineWorkflowOptions<
|
|
29
|
+
Input,
|
|
30
|
+
Output,
|
|
31
|
+
TSchema extends StandardSchemaV1 | undefined = undefined,
|
|
32
|
+
> = Omit<
|
|
33
|
+
WorkflowSpec<SchemaOutput<TSchema, Input>, Output, SchemaInput<TSchema, Input>>,
|
|
34
|
+
"name"
|
|
35
|
+
> & {
|
|
36
|
+
name?: string;
|
|
37
|
+
schedules?: {
|
|
38
|
+
name?: string;
|
|
39
|
+
expression: string;
|
|
40
|
+
input?: Executable<SchemaInput<TSchema, Input> | undefined>;
|
|
41
|
+
}[];
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
// 워크플로우 정의를 위한 데코레이터,
|
|
45
|
+
// 이것들은 syncer에서 한번에 load한 다음, WorkflowManager에서 synchronize를 통해 등록됨.
|
|
46
|
+
export function workflow<Input, Output, TSchema extends StandardSchemaV1 | undefined = undefined>(
|
|
47
|
+
options: DefineWorkflowOptions<Input, Output, TSchema>,
|
|
48
|
+
) {
|
|
49
|
+
return (fn: WorkflowFunction<SchemaOutput<TSchema, Input>, Output>) => {
|
|
50
|
+
const id = randomUUID();
|
|
51
|
+
const workflowName = options.name ?? inflection.underscore(fn.name);
|
|
52
|
+
|
|
53
|
+
const decorated: WorkflowMetadata = {
|
|
54
|
+
type: "workflow" as const,
|
|
55
|
+
id,
|
|
56
|
+
name: workflowName,
|
|
57
|
+
schema: options.schema,
|
|
58
|
+
version: options.version ?? null,
|
|
59
|
+
schedules: (options.schedules ?? []).map((schedule) => {
|
|
60
|
+
return {
|
|
61
|
+
name: schedule.name ?? `${workflowName}[${schedule.expression}]`,
|
|
62
|
+
expression: schedule.expression,
|
|
63
|
+
input: schedule.input,
|
|
64
|
+
};
|
|
65
|
+
}),
|
|
66
|
+
fn: fn as WorkflowFunction<unknown, unknown>,
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
return decorated;
|
|
70
|
+
};
|
|
71
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import type { DurationString, StepApi } from "@sonamu-kit/tasks/internal";
|
|
2
|
+
import inflection from "inflection";
|
|
3
|
+
|
|
4
|
+
export type StepFunction<TArgs extends unknown[], TResult> = (...args: TArgs) => TResult;
|
|
5
|
+
export type RunnableStep<TArgs extends unknown[], TResult> = {
|
|
6
|
+
run: StepFunction<TArgs, Promise<TResult>>;
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
export type MethodNames<T, TKey extends keyof T> = T[TKey] extends (
|
|
10
|
+
...args: infer _TArgs
|
|
11
|
+
) => infer _TResult
|
|
12
|
+
? TKey
|
|
13
|
+
: never;
|
|
14
|
+
|
|
15
|
+
export type MethodArguments<T, TKey extends keyof T> = T[TKey] extends (
|
|
16
|
+
...args: infer TArgs
|
|
17
|
+
) => unknown
|
|
18
|
+
? TArgs
|
|
19
|
+
: never;
|
|
20
|
+
|
|
21
|
+
export type MethodReturnType<T, TKey extends keyof T> = T[TKey] extends (
|
|
22
|
+
this: T,
|
|
23
|
+
...args: infer _TArgs
|
|
24
|
+
) => infer TResult
|
|
25
|
+
? TResult
|
|
26
|
+
: never;
|
|
27
|
+
|
|
28
|
+
export class StepWrapper {
|
|
29
|
+
readonly #stepApi: StepApi;
|
|
30
|
+
|
|
31
|
+
constructor(stepApi: StepApi) {
|
|
32
|
+
this.#stepApi = stepApi;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
get<
|
|
36
|
+
T,
|
|
37
|
+
TKey extends keyof T,
|
|
38
|
+
TArgs extends MethodArguments<T, TKey>,
|
|
39
|
+
TResult extends MethodReturnType<T, TKey>,
|
|
40
|
+
>(config: { name: string }, object: T, name: MethodNames<T, TKey>): RunnableStep<TArgs, TResult>;
|
|
41
|
+
get<
|
|
42
|
+
T,
|
|
43
|
+
TKey extends keyof T,
|
|
44
|
+
TArgs extends MethodArguments<T, TKey>,
|
|
45
|
+
TResult extends MethodReturnType<T, TKey>,
|
|
46
|
+
>(object: T, name: MethodNames<T, TKey>): RunnableStep<TArgs, TResult>;
|
|
47
|
+
get<
|
|
48
|
+
T,
|
|
49
|
+
TKey extends keyof T,
|
|
50
|
+
TArgs extends MethodArguments<T, TKey>,
|
|
51
|
+
TResult extends MethodReturnType<T, TKey>,
|
|
52
|
+
>(
|
|
53
|
+
...args: [{ name: string }, T, MethodNames<T, TKey>] | [T, MethodNames<T, TKey>]
|
|
54
|
+
): RunnableStep<TArgs, TResult> {
|
|
55
|
+
let config: { name: string };
|
|
56
|
+
let fn: StepFunction<TArgs, Exclude<TResult, never>>;
|
|
57
|
+
|
|
58
|
+
if (args.length === 2) {
|
|
59
|
+
const [rawObject, methodName] = args;
|
|
60
|
+
const method = rawObject[methodName] as CallableFunction;
|
|
61
|
+
config = { name: inflection.underscore(methodName.toString()) };
|
|
62
|
+
|
|
63
|
+
fn = (...args: TArgs) => method.bind(rawObject)(...args);
|
|
64
|
+
} else {
|
|
65
|
+
const [rawConfig, rawObject, name] = args;
|
|
66
|
+
const method = rawObject[name] as CallableFunction;
|
|
67
|
+
|
|
68
|
+
config = { name: rawConfig.name ?? inflection.underscore(name.toString()) };
|
|
69
|
+
fn = (...args: TArgs) => method.bind(rawObject)(...args);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return {
|
|
73
|
+
run: ((stepApi: StepApi) => {
|
|
74
|
+
return (...args: TArgs) => {
|
|
75
|
+
return stepApi.run(config, () => fn(...args));
|
|
76
|
+
};
|
|
77
|
+
})(this.#stepApi),
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
sleep(name: string, duration: DurationString) {
|
|
82
|
+
return this.#stepApi.sleep(name, duration);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
import { BackendPostgres, OpenWorkflow, type Worker } from "@sonamu-kit/tasks";
|
|
2
|
+
import type {
|
|
3
|
+
RunnableWorkflow,
|
|
4
|
+
SchemaInput,
|
|
5
|
+
SchemaOutput,
|
|
6
|
+
StandardSchemaV1,
|
|
7
|
+
StepApi,
|
|
8
|
+
WorkflowRunHandle,
|
|
9
|
+
WorkflowSpec,
|
|
10
|
+
} from "@sonamu-kit/tasks/internal";
|
|
11
|
+
import assert from "assert";
|
|
12
|
+
import type { Knex } from "knex";
|
|
13
|
+
import { schedule as cronSchedule, type ScheduledTask } from "node-cron";
|
|
14
|
+
import type { ZodObject } from "zod";
|
|
15
|
+
import type { Context } from "../api/context";
|
|
16
|
+
import { Sonamu } from "../api/sonamu";
|
|
17
|
+
import { Naite } from "../naite/naite";
|
|
18
|
+
import { createMockSSEFactory } from "../stream/sse";
|
|
19
|
+
import type { Executable } from "../types/types";
|
|
20
|
+
import type { WorkflowMetadata } from "./decorator";
|
|
21
|
+
import { StepWrapper } from "./step-wrapper";
|
|
22
|
+
|
|
23
|
+
export interface WorkflowOptions {
|
|
24
|
+
// worker에서 동시 실행할 태스크 수, 기본은 CPU 코어 수 - 1개
|
|
25
|
+
concurrency?: number;
|
|
26
|
+
|
|
27
|
+
// worker에서 사용할 pub/sub 여부, 기본은 true
|
|
28
|
+
usePubSub?: boolean;
|
|
29
|
+
|
|
30
|
+
// pub/sub으로 태스크를 수신했을 때 워크플로우 실행까지 지연 시간, 기본은 500ms
|
|
31
|
+
listenDelay?: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// Workflow 함수의 타입, @sonamu-kit/tasks와 다른 점은 step을 한번 감싼 형태.
|
|
35
|
+
export type WorkflowFunction<Input, Output> = (
|
|
36
|
+
params: Readonly<{
|
|
37
|
+
input: Input;
|
|
38
|
+
step: StepWrapper;
|
|
39
|
+
version: string | null;
|
|
40
|
+
}>,
|
|
41
|
+
) => Promise<Output> | Output;
|
|
42
|
+
|
|
43
|
+
// Workflow를 등록할 때를 위한 타입
|
|
44
|
+
export type WorkflowCreateOptions<
|
|
45
|
+
Input,
|
|
46
|
+
Output,
|
|
47
|
+
TSchema extends StandardSchemaV1 | undefined = undefined,
|
|
48
|
+
> = Omit<
|
|
49
|
+
WorkflowSpec<SchemaOutput<TSchema, Input>, Output, SchemaInput<TSchema, Input>>,
|
|
50
|
+
"version"
|
|
51
|
+
> & {
|
|
52
|
+
id: string;
|
|
53
|
+
version: string | null;
|
|
54
|
+
function: WorkflowFunction<SchemaOutput<TSchema, Input>, Output>;
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
export class WorkflowManager {
|
|
58
|
+
// backend 인스턴스
|
|
59
|
+
readonly #backend: BackendPostgres;
|
|
60
|
+
|
|
61
|
+
// OpenWorkflow 인스턴스
|
|
62
|
+
readonly #ow: OpenWorkflow;
|
|
63
|
+
|
|
64
|
+
// Worker 인스턴스 (없을 수 있으며, 이 때는 분산된 서버 환경에서 DB에 publish만 한다고 가정함)
|
|
65
|
+
#worker: Worker | null;
|
|
66
|
+
|
|
67
|
+
// 파일 경로 -> 파일에 정의된 워크플로우 메타데이터 목록
|
|
68
|
+
#workflowsMap: Map<string, WorkflowMetadata[]>;
|
|
69
|
+
|
|
70
|
+
// Task 이름 -> Task 정보 및 Input 값, Workflow ID
|
|
71
|
+
#scheduledTasks: Map<
|
|
72
|
+
string,
|
|
73
|
+
{
|
|
74
|
+
task: ScheduledTask;
|
|
75
|
+
inputFn: Executable<SchemaInput<unknown, unknown>>;
|
|
76
|
+
workflowId: string;
|
|
77
|
+
}
|
|
78
|
+
>;
|
|
79
|
+
|
|
80
|
+
private constructor(backend: BackendPostgres) {
|
|
81
|
+
this.#backend = backend;
|
|
82
|
+
this.#ow = new OpenWorkflow({ backend });
|
|
83
|
+
this.#worker = null;
|
|
84
|
+
this.#workflowsMap = new Map();
|
|
85
|
+
this.#scheduledTasks = new Map();
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* 정의된 워크플로우 및 워크플로우에 대한 scheduled tasks를 동기화합니다.
|
|
90
|
+
*/
|
|
91
|
+
async synchronize(workflowMap: Map<string, WorkflowMetadata[]>) {
|
|
92
|
+
// 1. 삭제된 파일은 일괄 삭제
|
|
93
|
+
await Promise.allSettled(
|
|
94
|
+
Array.from(this.#workflowsMap.entries())
|
|
95
|
+
.filter(([key]) => !workflowMap.has(key))
|
|
96
|
+
.flatMap(([_, workflows]) =>
|
|
97
|
+
workflows.map((workflow) => {
|
|
98
|
+
return Promise.allSettled([
|
|
99
|
+
...workflow.schedules.map((schedule) => this.unscheduleTask(schedule.name)),
|
|
100
|
+
this.unregisterWorkflow(workflow),
|
|
101
|
+
]);
|
|
102
|
+
}),
|
|
103
|
+
),
|
|
104
|
+
);
|
|
105
|
+
|
|
106
|
+
// 2. 새로 추가된 파일은 일괄 등록
|
|
107
|
+
await Promise.allSettled(
|
|
108
|
+
Array.from(workflowMap.entries())
|
|
109
|
+
.filter(([key]) => !this.#workflowsMap.has(key))
|
|
110
|
+
.flatMap(([_, workflows]) =>
|
|
111
|
+
workflows.map((workflow) => {
|
|
112
|
+
this.registerWorkflow({
|
|
113
|
+
id: workflow.id,
|
|
114
|
+
name: workflow.name,
|
|
115
|
+
version: workflow.version ?? null,
|
|
116
|
+
schema: workflow.schema,
|
|
117
|
+
function: workflow.fn,
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
return Promise.allSettled(
|
|
121
|
+
workflow.schedules.map((schedule) => this.scheduleTask(workflow, schedule)),
|
|
122
|
+
);
|
|
123
|
+
}),
|
|
124
|
+
),
|
|
125
|
+
);
|
|
126
|
+
|
|
127
|
+
// 3. 기존과 다른 것을 diff친 다음, 삭제 후 재등록
|
|
128
|
+
await Promise.allSettled(
|
|
129
|
+
Array.from(workflowMap.entries())
|
|
130
|
+
.filter(([key]) => this.#workflowsMap.has(key))
|
|
131
|
+
.map<[string, [WorkflowMetadata[], WorkflowMetadata[]]]>(([key, newWorkflows]) => {
|
|
132
|
+
const previousWorkflows = this.#workflowsMap.get(key);
|
|
133
|
+
assert(previousWorkflows, "previous workflows not found");
|
|
134
|
+
return [key, [previousWorkflows, newWorkflows]];
|
|
135
|
+
})
|
|
136
|
+
.map(async ([_, [previousWorkflows, newWorkflows]]) => {
|
|
137
|
+
// 기존 것들을 삭제부터 해야함.
|
|
138
|
+
await Promise.allSettled(
|
|
139
|
+
previousWorkflows
|
|
140
|
+
.filter((prevItem) => !newWorkflows.some((newItem) => newItem.id === prevItem.id))
|
|
141
|
+
.map((prevItem) => {
|
|
142
|
+
return Promise.allSettled([
|
|
143
|
+
...prevItem.schedules.map((schedule) => this.unscheduleTask(schedule.name)),
|
|
144
|
+
this.unregisterWorkflow(prevItem),
|
|
145
|
+
]);
|
|
146
|
+
}),
|
|
147
|
+
);
|
|
148
|
+
|
|
149
|
+
// 새로 추가된 것들을 등록
|
|
150
|
+
await Promise.allSettled(
|
|
151
|
+
newWorkflows
|
|
152
|
+
.filter(
|
|
153
|
+
(newItem) => !previousWorkflows.some((prevItem) => prevItem.id === newItem.id),
|
|
154
|
+
)
|
|
155
|
+
.map(async (newItem) => {
|
|
156
|
+
this.registerWorkflow({
|
|
157
|
+
id: newItem.id,
|
|
158
|
+
name: newItem.name,
|
|
159
|
+
version: newItem.version ?? null,
|
|
160
|
+
schema: newItem.schema,
|
|
161
|
+
function: newItem.fn,
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
return Promise.allSettled(
|
|
165
|
+
newItem.schedules.map((schedule) => this.scheduleTask(newItem, schedule)),
|
|
166
|
+
);
|
|
167
|
+
}),
|
|
168
|
+
);
|
|
169
|
+
}),
|
|
170
|
+
);
|
|
171
|
+
|
|
172
|
+
this.#workflowsMap = workflowMap;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// 워크플로우를 실행
|
|
176
|
+
run<Input, Output, TSchema extends StandardSchemaV1 | undefined = undefined>(
|
|
177
|
+
options: Omit<WorkflowCreateOptions<Input, Output, TSchema>, "function" | "schema" | "id">,
|
|
178
|
+
input: SchemaInput<TSchema, Input>,
|
|
179
|
+
): Promise<WorkflowRunHandle<Output>> {
|
|
180
|
+
return this.#ow.runWorkflow(
|
|
181
|
+
{
|
|
182
|
+
name: options.name,
|
|
183
|
+
version: options.version ?? undefined,
|
|
184
|
+
},
|
|
185
|
+
input,
|
|
186
|
+
);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// cron task를 등록
|
|
190
|
+
async scheduleTask(
|
|
191
|
+
workflow: Pick<WorkflowMetadata, "id" | "name" | "version">,
|
|
192
|
+
schedule: WorkflowMetadata["schedules"][number],
|
|
193
|
+
) {
|
|
194
|
+
const task = cronSchedule(
|
|
195
|
+
schedule.expression,
|
|
196
|
+
(async (
|
|
197
|
+
{ name, version }: Pick<WorkflowMetadata, "name" | "version">,
|
|
198
|
+
{ input }: WorkflowMetadata["schedules"][number],
|
|
199
|
+
) => {
|
|
200
|
+
const inputData = await (typeof input === "function"
|
|
201
|
+
? Promise.resolve(input())
|
|
202
|
+
: Promise.resolve(input));
|
|
203
|
+
|
|
204
|
+
return this.run({ name, version: version ?? null }, inputData);
|
|
205
|
+
}).bind(this, workflow, schedule),
|
|
206
|
+
{
|
|
207
|
+
name: schedule.name,
|
|
208
|
+
timezone: Sonamu.config.api.timezone,
|
|
209
|
+
noOverlap: false,
|
|
210
|
+
},
|
|
211
|
+
);
|
|
212
|
+
|
|
213
|
+
this.#scheduledTasks.set(schedule.name, {
|
|
214
|
+
task,
|
|
215
|
+
inputFn: schedule.input,
|
|
216
|
+
workflowId: workflow.id,
|
|
217
|
+
});
|
|
218
|
+
await task.start();
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// cron task를 중지
|
|
222
|
+
async unscheduleTask(name: string) {
|
|
223
|
+
const taskItem = this.#scheduledTasks.get(name);
|
|
224
|
+
if (!taskItem) {
|
|
225
|
+
console.error("scheduled task not found", name);
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
this.#scheduledTasks.delete(name);
|
|
230
|
+
await taskItem.task.stop();
|
|
231
|
+
await taskItem.task.destroy();
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// 워크플로우를 등록, 관련된 스케줄은 별도로 등록해야 함.
|
|
235
|
+
registerWorkflow<Input, Output, TSchema extends StandardSchemaV1 | undefined = undefined>(
|
|
236
|
+
options: WorkflowCreateOptions<Input, Output, TSchema>,
|
|
237
|
+
): RunnableWorkflow<SchemaOutput<TSchema, Input>, Output, SchemaInput<TSchema, Input>> {
|
|
238
|
+
const fn = async (
|
|
239
|
+
params: Readonly<{
|
|
240
|
+
input: SchemaOutput<TSchema, Input>;
|
|
241
|
+
step: StepApi;
|
|
242
|
+
version: string | null;
|
|
243
|
+
}>,
|
|
244
|
+
) => {
|
|
245
|
+
const baseContext = {
|
|
246
|
+
request: null,
|
|
247
|
+
reply: null,
|
|
248
|
+
headers: {},
|
|
249
|
+
createSSE: (schema: ZodObject) => createMockSSEFactory(schema),
|
|
250
|
+
naiteStore: Naite.createStore(),
|
|
251
|
+
user: null,
|
|
252
|
+
passport: {
|
|
253
|
+
login: async () => {},
|
|
254
|
+
logout: () => {},
|
|
255
|
+
},
|
|
256
|
+
} as unknown as Context;
|
|
257
|
+
|
|
258
|
+
const contextProvider = Sonamu.config.tasks?.contextProvider;
|
|
259
|
+
const context: Context = contextProvider
|
|
260
|
+
? await Promise.resolve(contextProvider(baseContext))
|
|
261
|
+
: baseContext;
|
|
262
|
+
|
|
263
|
+
const step = new StepWrapper(params.step);
|
|
264
|
+
return Sonamu.asyncLocalStorage.run({ context }, () =>
|
|
265
|
+
options.function({ input: params.input, step, version: params.version }),
|
|
266
|
+
);
|
|
267
|
+
};
|
|
268
|
+
|
|
269
|
+
const workflow = this.#ow.defineWorkflow(
|
|
270
|
+
{
|
|
271
|
+
name: options.name,
|
|
272
|
+
version: options.version ?? undefined,
|
|
273
|
+
schema: options.schema,
|
|
274
|
+
},
|
|
275
|
+
fn,
|
|
276
|
+
);
|
|
277
|
+
return workflow;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
// 워크플로우를 등록 해제, 관련된 스케줄은 별도로 해제해야 함.
|
|
281
|
+
async unregisterWorkflow(workflow: Pick<WorkflowMetadata, "name" | "version" | "id">) {
|
|
282
|
+
this.#ow.unregisterWorkflow(workflow.name, workflow.version ?? null);
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
// Worker를 설정 후 시작
|
|
286
|
+
async setupWorker(options: WorkflowOptions) {
|
|
287
|
+
this.#worker = this.#ow.newWorker(options);
|
|
288
|
+
await this.#worker.start();
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// Worker를 중지
|
|
292
|
+
async stopWorker() {
|
|
293
|
+
await this.#worker?.stop();
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// cron task들을 모두 중지
|
|
297
|
+
async stopSchedules() {
|
|
298
|
+
await Promise.allSettled(
|
|
299
|
+
Array.from(this.#scheduledTasks.values()).map(({ task }) => Promise.resolve(task.stop())),
|
|
300
|
+
);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// cron task들을 모두 정리
|
|
304
|
+
async destroySchedules() {
|
|
305
|
+
await Promise.allSettled(
|
|
306
|
+
Array.from(this.#scheduledTasks.values()).map(({ task }) => Promise.resolve(task.destroy())),
|
|
307
|
+
);
|
|
308
|
+
this.#scheduledTasks.clear();
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
async destroy() {
|
|
312
|
+
await this.stopSchedules();
|
|
313
|
+
await this.stopWorker();
|
|
314
|
+
await this.destroySchedules();
|
|
315
|
+
await this.#backend.stop();
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
[Symbol.asyncDispose]() {
|
|
319
|
+
return this.destroy();
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// BackendPostgres에서 처리하는 것들이 있어서 Knex 커넥션이 아니라 설정값을 넣어줘야함.
|
|
323
|
+
static async create(
|
|
324
|
+
dbConf: Knex.Config,
|
|
325
|
+
runMigrations: boolean = true,
|
|
326
|
+
): Promise<WorkflowManager> {
|
|
327
|
+
const backend = await BackendPostgres.connect(dbConf, { runMigrations });
|
|
328
|
+
return new WorkflowManager(backend);
|
|
329
|
+
}
|
|
330
|
+
}
|
|
@@ -190,6 +190,14 @@ export class Template__generated extends Template {
|
|
|
190
190
|
.map((prop) => (prop.type === "relation" ? `${prop.name}_id` : prop.name))
|
|
191
191
|
.concat("id");
|
|
192
192
|
|
|
193
|
+
/**
|
|
194
|
+
* hasVector props
|
|
195
|
+
* - vector 타입인 컬럼
|
|
196
|
+
*/
|
|
197
|
+
const hasVectorColumns = entity.props
|
|
198
|
+
.filter((prop) => prop.type === "vector" || prop.type === "vector[]")
|
|
199
|
+
.map((prop) => prop.name);
|
|
200
|
+
|
|
193
201
|
/**
|
|
194
202
|
* generated props
|
|
195
203
|
* - generated 속성이 있는 컬럼 (INSERT/UPDATE 시 값 제공 불가)
|
|
@@ -202,7 +210,8 @@ export class Template__generated extends Template {
|
|
|
202
210
|
fulltextColumns.length > 0 ||
|
|
203
211
|
virtualProps.length > 0 ||
|
|
204
212
|
hasDefaultColumns.length > 0 ||
|
|
205
|
-
generatedColumns.length > 0
|
|
213
|
+
generatedColumns.length > 0 ||
|
|
214
|
+
hasVectorColumns.length > 0;
|
|
206
215
|
|
|
207
216
|
const lines = [
|
|
208
217
|
`export const ${schemaName} = ${schemaBody};`,
|
|
@@ -217,15 +226,20 @@ export class Template__generated extends Template {
|
|
|
217
226
|
(virtualProps.length > 0
|
|
218
227
|
? `readonly __virtual__: readonly [${virtualProps.map((prop) => `"${prop}"`).join(", ")}],`
|
|
219
228
|
: "") +
|
|
229
|
+
(hasDefaultColumns.length > 0
|
|
230
|
+
? `readonly __hasDefault__: readonly [${hasDefaultColumns
|
|
231
|
+
.map((col) => `"${col}"`)
|
|
232
|
+
.join(", ")}],`
|
|
233
|
+
: "") +
|
|
220
234
|
(
|
|
221
|
-
|
|
222
|
-
? `readonly
|
|
235
|
+
generatedColumns.length > 0
|
|
236
|
+
? `readonly __generated__: readonly [${generatedColumns
|
|
223
237
|
.map((col) => `"${col}"`)
|
|
224
238
|
.join(", ")}],`
|
|
225
239
|
: ""
|
|
226
240
|
) +
|
|
227
|
-
(
|
|
228
|
-
? `readonly
|
|
241
|
+
(hasVectorColumns.length > 0
|
|
242
|
+
? `readonly __vector__: readonly [${hasVectorColumns
|
|
229
243
|
.map((col) => `"${col}"`)
|
|
230
244
|
.join(", ")}],`
|
|
231
245
|
: "")
|
|
@@ -265,7 +279,6 @@ export class Template__generated extends Template {
|
|
|
265
279
|
const filterBody = propNodes
|
|
266
280
|
.map((propNode) => propNodeToZodTypeDef(propNode, importKeys))
|
|
267
281
|
.join("\n");
|
|
268
|
-
|
|
269
282
|
const schemaBody = `
|
|
270
283
|
z.object({
|
|
271
284
|
num: z.number().int().nonnegative(),
|
|
@@ -105,7 +105,7 @@ class ${entityId}ModelClass extends BaseModelClass<
|
|
|
105
105
|
search: "${def.search}" as const,
|
|
106
106
|
orderBy: "${def.orderBy}" as const,
|
|
107
107
|
...rawParams,
|
|
108
|
-
};
|
|
108
|
+
} satisfies ${entityId}ListParams;
|
|
109
109
|
|
|
110
110
|
// build queries
|
|
111
111
|
const { qb, onSubset: _ } = this.getSubsetQueries(subset);
|
|
@@ -47,6 +47,7 @@ import {
|
|
|
47
47
|
isRelationProp,
|
|
48
48
|
isStringArrayProp,
|
|
49
49
|
isStringSingleProp,
|
|
50
|
+
isTsVectorProp,
|
|
50
51
|
isUuidArrayProp,
|
|
51
52
|
isUuidSingleProp,
|
|
52
53
|
isVectorArrayProp,
|
|
@@ -215,6 +216,8 @@ export function propToZodTypeDef(prop: EntityProp, injectImportKeys: string[]):
|
|
|
215
216
|
stmt = `${prop.name}: z.array(z.number())`;
|
|
216
217
|
} else if (isVectorArrayProp(prop)) {
|
|
217
218
|
stmt = `${prop.name}: z.array(z.array(z.number()))`;
|
|
219
|
+
} else if (isTsVectorProp(prop)) {
|
|
220
|
+
stmt = `${prop.name}: z.string()`;
|
|
218
221
|
} else if (isVirtualProp(prop)) {
|
|
219
222
|
stmt = `${prop.name}: ${prop.id}`;
|
|
220
223
|
injectImportKeys.push(prop.id);
|
package/src/types/types.ts
CHANGED
|
@@ -13,7 +13,7 @@ export function zArrayable<T extends z.ZodTypeAny>(shape: T): z.ZodUnion<[T, z.Z
|
|
|
13
13
|
export type DistributiveOmit<T, K extends keyof any> = T extends any ? Omit<T, K> : never;
|
|
14
14
|
|
|
15
15
|
/*
|
|
16
|
-
Model-
|
|
16
|
+
Model-Definition
|
|
17
17
|
*/
|
|
18
18
|
export type GeneratedColumnType = "STORED" | "VIRTUAL";
|
|
19
19
|
export type GeneratedColumn = {
|
|
@@ -227,7 +227,7 @@ export type EntityIndex = {
|
|
|
227
227
|
type: "index" | "unique" | "hnsw" | "ivfflat";
|
|
228
228
|
columns: EntityIndexColumn[];
|
|
229
229
|
name: string;
|
|
230
|
-
using?: "btree" | "hash" | "gin" | "gist";
|
|
230
|
+
using?: "btree" | "hash" | "gin" | "gist" | "pgroonga";
|
|
231
231
|
nullsNotDistinct?: boolean; // unique index only
|
|
232
232
|
/**
|
|
233
233
|
* HNSW (Hierarchical Navigable Small World) 인덱스: 각 노드의 최대 연결 수
|
|
@@ -422,6 +422,9 @@ export function isVectorArrayProp(p: unknown): p is VectorArrayProp {
|
|
|
422
422
|
export function isVectorProp(p: unknown): p is VectorProp | VectorArrayProp {
|
|
423
423
|
return isVectorSingleProp(p) || isVectorArrayProp(p);
|
|
424
424
|
}
|
|
425
|
+
export function isTsVectorProp(p: unknown): p is TsVectorProp {
|
|
426
|
+
return (p as TsVectorProp)?.type === "tsvector";
|
|
427
|
+
}
|
|
425
428
|
export function isRelationProp(p: unknown): p is RelationProp {
|
|
426
429
|
return (p as RelationProp)?.type === "relation";
|
|
427
430
|
}
|
|
@@ -489,6 +492,19 @@ export type SubsetQuery = {
|
|
|
489
492
|
export const SonamuQueryMode = z.enum(["both", "list", "count"]);
|
|
490
493
|
export type SonamuQueryMode = z.infer<typeof SonamuQueryMode>;
|
|
491
494
|
|
|
495
|
+
/* Semantic Query */
|
|
496
|
+
export const SonamuSemanticParams = z
|
|
497
|
+
.object({
|
|
498
|
+
semanticQuery: z.object({
|
|
499
|
+
embedding: z.array(z.number()).min(1024).max(1024),
|
|
500
|
+
threshold: z.number().optional(),
|
|
501
|
+
method: z.enum(["cosine", "l2", "inner_product"]).optional(),
|
|
502
|
+
which: z.string(),
|
|
503
|
+
}),
|
|
504
|
+
})
|
|
505
|
+
.partial();
|
|
506
|
+
export type SonamuSemanticParams = z.infer<typeof SonamuSemanticParams>;
|
|
507
|
+
|
|
492
508
|
/* Knex Migration */
|
|
493
509
|
export type KnexError = {
|
|
494
510
|
code: string;
|
|
@@ -554,7 +570,7 @@ export type MigrationIndex = {
|
|
|
554
570
|
type: "unique" | "index" | "hnsw" | "ivfflat";
|
|
555
571
|
columns: EntityIndexColumn[];
|
|
556
572
|
name: string;
|
|
557
|
-
using?: "btree" | "hash" | "gin" | "gist";
|
|
573
|
+
using?: "btree" | "hash" | "gin" | "gist" | "pgroonga";
|
|
558
574
|
nullsNotDistinct?: boolean;
|
|
559
575
|
/** HNSW (Hierarchical Navigable Small World): 각 노드의 최대 연결 수 */
|
|
560
576
|
m?: number;
|
|
@@ -1140,7 +1156,7 @@ const EntityIndexSchema = z
|
|
|
1140
1156
|
type: z.enum(["index", "unique", "hnsw", "ivfflat"]),
|
|
1141
1157
|
columns: z.array(EntityIndexColumnSchema),
|
|
1142
1158
|
name: z.string().min(1).max(63),
|
|
1143
|
-
using: z.enum(["btree", "hash", "gin", "gist"]).optional(),
|
|
1159
|
+
using: z.enum(["btree", "hash", "gin", "gist", "pgroonga"]).optional(),
|
|
1144
1160
|
nullsNotDistinct: z.boolean().optional(),
|
|
1145
1161
|
m: z.number().optional(),
|
|
1146
1162
|
efConstruction: z.number().optional(),
|
|
@@ -1331,6 +1347,9 @@ export type ManyToManyBaseSchema<FromIdKey extends string, ToIdKey extends strin
|
|
|
1331
1347
|
[K in `${ToIdKey}_id`]: number;
|
|
1332
1348
|
};
|
|
1333
1349
|
|
|
1350
|
+
// 객체, 함수, 비동기 함수를 모두 포괄하는 타입
|
|
1351
|
+
export type Executable<T> = T | Promise<T> | (() => T) | (() => Promise<T>);
|
|
1352
|
+
|
|
1334
1353
|
export type SonamuFastifyConfig = {
|
|
1335
1354
|
contextProvider: (
|
|
1336
1355
|
defaultContext: Pick<Context, "request" | "reply" | "headers" | "createSSE" | "naiteStore"> &
|
package/src/utils/formatter.ts
CHANGED
|
@@ -105,5 +105,12 @@ export function formatCode(code: string, parser: "typescript" | "json", filePath
|
|
|
105
105
|
}
|
|
106
106
|
Naite.t("formatCode:linted", linted);
|
|
107
107
|
|
|
108
|
-
|
|
108
|
+
// 포맷팅 한 번 더 (import 구문에 type 키워드 추가되는 경우 maxWidth 초과로 인한 에러 발생)
|
|
109
|
+
const formattedAgain = biome.formatContent(projectKey, linted.content, { filePath });
|
|
110
|
+
if (formattedAgain.diagnostics.filter((d) => d.severity === "error").length > 0) {
|
|
111
|
+
console.error(formattedAgain.diagnostics);
|
|
112
|
+
throw new Error("Biome format error");
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return formattedAgain.content;
|
|
109
116
|
}
|