sonamu 0.7.4 → 0.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/config.d.ts +1 -4
- package/dist/api/config.d.ts.map +1 -1
- package/dist/api/config.js +1 -1
- package/dist/api/sonamu.d.ts +2 -0
- package/dist/api/sonamu.d.ts.map +1 -1
- package/dist/api/sonamu.js +19 -47
- package/dist/bin/cli.js +6 -6
- package/dist/database/base-model.d.ts +1 -1
- package/dist/database/base-model.d.ts.map +1 -1
- package/dist/database/base-model.js +15 -4
- package/dist/database/code-generator.d.ts.map +1 -1
- package/dist/database/code-generator.js +3 -3
- package/dist/database/db.d.ts.map +1 -1
- package/dist/database/db.js +1 -1
- package/dist/database/puri-wrapper.d.ts +11 -11
- package/dist/database/puri-wrapper.d.ts.map +1 -1
- package/dist/database/puri-wrapper.js +7 -11
- package/dist/database/puri.d.ts +36 -17
- package/dist/database/puri.d.ts.map +1 -1
- package/dist/database/puri.js +54 -7
- package/dist/database/puri.types.d.ts +54 -17
- package/dist/database/puri.types.d.ts.map +1 -1
- package/dist/database/puri.types.js +2 -4
- package/dist/database/puri.types.test-d.js +129 -0
- package/dist/database/upsert-builder.d.ts +16 -10
- package/dist/database/upsert-builder.d.ts.map +1 -1
- package/dist/database/upsert-builder.js +10 -19
- package/dist/entity/entity-manager.d.ts +113 -22
- package/dist/entity/entity-manager.d.ts.map +1 -1
- package/dist/entity/entity-manager.js +1 -1
- package/dist/entity/entity.d.ts +34 -0
- package/dist/entity/entity.d.ts.map +1 -1
- package/dist/entity/entity.js +110 -37
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -2
- package/dist/migration/code-generation.d.ts.map +1 -1
- package/dist/migration/code-generation.js +341 -149
- package/dist/migration/migration-set.d.ts.map +1 -1
- package/dist/migration/migration-set.js +21 -5
- package/dist/migration/migrator.d.ts.map +1 -1
- package/dist/migration/migrator.js +7 -1
- package/dist/migration/postgresql-schema-reader.d.ts +11 -1
- package/dist/migration/postgresql-schema-reader.d.ts.map +1 -1
- package/dist/migration/postgresql-schema-reader.js +111 -10
- package/dist/syncer/syncer.d.ts.map +1 -1
- package/dist/syncer/syncer.js +7 -4
- package/dist/template/implementations/generated.template.d.ts.map +1 -1
- package/dist/template/implementations/generated.template.js +12 -2
- package/dist/template/implementations/generated_sso.template.d.ts +3 -3
- package/dist/template/implementations/generated_sso.template.d.ts.map +1 -1
- package/dist/template/implementations/generated_sso.template.js +50 -2
- package/dist/template/implementations/model.template.d.ts.map +1 -1
- package/dist/template/implementations/model.template.js +20 -15
- package/dist/template/implementations/model_test.template.js +4 -4
- package/dist/template/implementations/service.template.d.ts.map +1 -1
- package/dist/template/implementations/service.template.js +2 -2
- package/dist/template/implementations/view_enums_dropdown.template.js +2 -2
- package/dist/template/implementations/view_enums_select.template.js +2 -2
- package/dist/template/implementations/view_form.template.d.ts.map +1 -1
- package/dist/template/implementations/view_form.template.js +12 -9
- package/dist/template/implementations/view_id_async_select.template.js +4 -4
- package/dist/template/implementations/view_list.template.d.ts.map +1 -1
- package/dist/template/implementations/view_list.template.js +12 -9
- package/dist/template/implementations/view_search_input.template.js +2 -2
- package/dist/template/template.js +2 -2
- package/dist/template/zod-converter.d.ts.map +1 -1
- package/dist/template/zod-converter.js +17 -2
- package/dist/testing/fixture-manager.d.ts +2 -1
- package/dist/testing/fixture-manager.d.ts.map +1 -1
- package/dist/testing/fixture-manager.js +29 -29
- package/dist/types/types.d.ts +593 -68
- package/dist/types/types.d.ts.map +1 -1
- package/dist/types/types.js +113 -9
- package/dist/vector/chunking.d.ts +25 -0
- package/dist/vector/chunking.d.ts.map +1 -0
- package/dist/vector/chunking.js +97 -0
- package/dist/vector/config.d.ts +12 -0
- package/dist/vector/config.d.ts.map +1 -0
- package/dist/vector/config.js +83 -0
- package/dist/vector/embedding.d.ts +42 -0
- package/dist/vector/embedding.d.ts.map +1 -0
- package/dist/vector/embedding.js +147 -0
- package/dist/vector/types.d.ts +105 -0
- package/dist/vector/types.d.ts.map +1 -0
- package/dist/vector/types.js +5 -0
- package/dist/vector/vector-search.d.ts +47 -0
- package/dist/vector/vector-search.d.ts.map +1 -0
- package/dist/vector/vector-search.js +176 -0
- package/package.json +9 -8
- package/src/api/config.ts +0 -4
- package/src/api/sonamu.ts +21 -36
- package/src/bin/cli.ts +5 -5
- package/src/database/base-model.ts +20 -11
- package/src/database/code-generator.ts +6 -2
- package/src/database/db.ts +1 -0
- package/src/database/puri-wrapper.ts +22 -16
- package/src/database/puri.ts +150 -27
- package/src/database/puri.types.test-d.ts +457 -0
- package/src/database/puri.types.ts +231 -33
- package/src/database/upsert-builder.ts +43 -34
- package/src/entity/entity-manager.ts +2 -2
- package/src/entity/entity.ts +134 -44
- package/src/index.ts +6 -0
- package/src/migration/code-generation.ts +377 -174
- package/src/migration/migration-set.ts +22 -3
- package/src/migration/migrator.ts +6 -0
- package/src/migration/postgresql-schema-reader.ts +121 -21
- package/src/syncer/syncer.ts +6 -3
- package/src/template/implementations/generated.template.ts +51 -9
- package/src/template/implementations/generated_sso.template.ts +71 -2
- package/src/template/implementations/model.template.ts +25 -15
- package/src/template/implementations/model_test.template.ts +3 -3
- package/src/template/implementations/service.template.ts +5 -1
- package/src/template/implementations/view_enums_dropdown.template.ts +1 -1
- package/src/template/implementations/view_enums_select.template.ts +1 -1
- package/src/template/implementations/view_form.template.ts +11 -8
- package/src/template/implementations/view_id_async_select.template.ts +3 -3
- package/src/template/implementations/view_list.template.ts +11 -8
- package/src/template/implementations/view_search_input.template.ts +1 -1
- package/src/template/template.ts +1 -1
- package/src/template/zod-converter.ts +20 -0
- package/src/testing/fixture-manager.ts +31 -30
- package/src/types/types.ts +226 -48
- package/src/vector/chunking.ts +115 -0
- package/src/vector/config.ts +68 -0
- package/src/vector/embedding.ts +193 -0
- package/src/vector/types.ts +122 -0
- package/src/vector/vector-search.ts +261 -0
- package/dist/template/implementations/view_enums_buttonset.template.d.ts +0 -17
- package/dist/template/implementations/view_enums_buttonset.template.d.ts.map +0 -1
- package/dist/template/implementations/view_enums_buttonset.template.js +0 -31
- package/dist/template/implementations/view_list_columns.template.d.ts +0 -17
- package/dist/template/implementations/view_list_columns.template.d.ts.map +0 -1
- package/dist/template/implementations/view_list_columns.template.js +0 -49
- package/src/template/implementations/view_enums_buttonset.template.ts +0 -34
- package/src/template/implementations/view_list_columns.template.ts +0 -53
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
import { Sonamu } from "../api/sonamu";
|
|
2
|
+
import { DEFAULT_VECTOR_CONFIG } from "./config";
|
|
3
|
+
import type {
|
|
4
|
+
EmbeddingProvider,
|
|
5
|
+
EmbeddingResult,
|
|
6
|
+
ProgressCallback,
|
|
7
|
+
VectorConfig,
|
|
8
|
+
VectorInputType,
|
|
9
|
+
} from "./types";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* 임베딩 클라이언트
|
|
13
|
+
* Voyage AI와 OpenAI 임베딩을 통합 지원
|
|
14
|
+
*/
|
|
15
|
+
export class Embedding {
|
|
16
|
+
private config: VectorConfig;
|
|
17
|
+
|
|
18
|
+
constructor(config: Partial<VectorConfig> = {}) {
|
|
19
|
+
this.config = {
|
|
20
|
+
voyage: { ...DEFAULT_VECTOR_CONFIG.voyage, ...config.voyage },
|
|
21
|
+
openai: { ...DEFAULT_VECTOR_CONFIG.openai, ...config.openai },
|
|
22
|
+
chunking: { ...DEFAULT_VECTOR_CONFIG.chunking, ...config.chunking },
|
|
23
|
+
search: { ...DEFAULT_VECTOR_CONFIG.search, ...config.search },
|
|
24
|
+
pgvector: { ...DEFAULT_VECTOR_CONFIG.pgvector, ...config.pgvector },
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* 텍스트 임베딩 생성
|
|
30
|
+
* @param texts - 임베딩할 텍스트 배열
|
|
31
|
+
* @param provider - 'voyage' | 'openai'
|
|
32
|
+
* @param inputType - 'document' | 'query' (Voyage AI만 해당)
|
|
33
|
+
*/
|
|
34
|
+
async embed(
|
|
35
|
+
texts: string[],
|
|
36
|
+
provider: EmbeddingProvider,
|
|
37
|
+
inputType: VectorInputType = "document"
|
|
38
|
+
): Promise<EmbeddingResult[]> {
|
|
39
|
+
if (provider === "voyage") {
|
|
40
|
+
return this.embedVoyage(texts, inputType);
|
|
41
|
+
} else {
|
|
42
|
+
return this.embedOpenAI(texts);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* 단일 텍스트 임베딩 (편의 메서드)
|
|
48
|
+
*/
|
|
49
|
+
async embedOne(
|
|
50
|
+
text: string,
|
|
51
|
+
provider: EmbeddingProvider,
|
|
52
|
+
inputType: VectorInputType = "document"
|
|
53
|
+
): Promise<EmbeddingResult> {
|
|
54
|
+
const results = await this.embed([text], provider, inputType);
|
|
55
|
+
return results[0];
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Voyage AI 임베딩
|
|
60
|
+
*/
|
|
61
|
+
private async embedVoyage(
|
|
62
|
+
texts: string[],
|
|
63
|
+
inputType: VectorInputType
|
|
64
|
+
): Promise<EmbeddingResult[]> {
|
|
65
|
+
const voyageConfig = this.config.voyage;
|
|
66
|
+
|
|
67
|
+
// config에서 설정된 apiKey 우선, 없으면 Sonamu.secrets에서 로드
|
|
68
|
+
const apiKey = voyageConfig.apiKey || Sonamu.secrets?.voyage_api_key;
|
|
69
|
+
if (!apiKey) {
|
|
70
|
+
throw new Error(
|
|
71
|
+
"VOYAGE_API_KEY가 설정되지 않았습니다. 환경변수를 확인하세요."
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const response = await fetch(voyageConfig.baseUrl, {
|
|
76
|
+
method: "POST",
|
|
77
|
+
headers: {
|
|
78
|
+
"Content-Type": "application/json",
|
|
79
|
+
Authorization: `Bearer ${apiKey}`,
|
|
80
|
+
},
|
|
81
|
+
body: JSON.stringify({
|
|
82
|
+
input: texts,
|
|
83
|
+
model: voyageConfig.model,
|
|
84
|
+
input_type: inputType,
|
|
85
|
+
}),
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
if (!response.ok) {
|
|
89
|
+
const error = await response.text();
|
|
90
|
+
throw new Error(`Voyage API error: ${response.status} - ${error}`);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const data = await response.json();
|
|
94
|
+
|
|
95
|
+
return data.data.map((item: { embedding: number[] }) => ({
|
|
96
|
+
embedding: item.embedding,
|
|
97
|
+
model: voyageConfig.model,
|
|
98
|
+
tokenCount: data.usage?.total_tokens || 0,
|
|
99
|
+
}));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* OpenAI 임베딩
|
|
104
|
+
*/
|
|
105
|
+
private async embedOpenAI(texts: string[]): Promise<EmbeddingResult[]> {
|
|
106
|
+
const openaiConfig = this.config.openai;
|
|
107
|
+
|
|
108
|
+
// config에서 설정된 apiKey 우선, 없으면 Sonamu.secrets에서 로드
|
|
109
|
+
const apiKey = openaiConfig.apiKey || Sonamu.secrets?.openai_api_key;
|
|
110
|
+
if (!apiKey) {
|
|
111
|
+
throw new Error(
|
|
112
|
+
"OPENAI_API_KEY가 설정되지 않았습니다. 환경변수를 확인하세요."
|
|
113
|
+
);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const response = await fetch(openaiConfig.baseUrl, {
|
|
117
|
+
method: "POST",
|
|
118
|
+
headers: {
|
|
119
|
+
"Content-Type": "application/json",
|
|
120
|
+
Authorization: `Bearer ${apiKey}`,
|
|
121
|
+
},
|
|
122
|
+
body: JSON.stringify({
|
|
123
|
+
input: texts,
|
|
124
|
+
model: openaiConfig.model,
|
|
125
|
+
}),
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
if (!response.ok) {
|
|
129
|
+
const error = await response.text();
|
|
130
|
+
throw new Error(`OpenAI API error: ${response.status} - ${error}`);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const data = await response.json();
|
|
134
|
+
|
|
135
|
+
return data.data.map((item: { embedding: number[] }) => ({
|
|
136
|
+
embedding: item.embedding,
|
|
137
|
+
model: openaiConfig.model,
|
|
138
|
+
tokenCount: data.usage?.total_tokens || 0,
|
|
139
|
+
}));
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* 배치 임베딩 (대량 처리)
|
|
144
|
+
*/
|
|
145
|
+
async embedBatch(
|
|
146
|
+
texts: string[],
|
|
147
|
+
provider: EmbeddingProvider,
|
|
148
|
+
inputType: VectorInputType = "document",
|
|
149
|
+
onProgress?: ProgressCallback
|
|
150
|
+
): Promise<EmbeddingResult[]> {
|
|
151
|
+
const batchSize =
|
|
152
|
+
provider === "voyage"
|
|
153
|
+
? this.config.voyage.batchSize
|
|
154
|
+
: this.config.openai.batchSize;
|
|
155
|
+
|
|
156
|
+
const results: EmbeddingResult[] = [];
|
|
157
|
+
|
|
158
|
+
for (let i = 0; i < texts.length; i += batchSize) {
|
|
159
|
+
const batch = texts.slice(i, i + batchSize);
|
|
160
|
+
const batchResults = await this.embed(batch, provider, inputType);
|
|
161
|
+
results.push(...batchResults);
|
|
162
|
+
|
|
163
|
+
onProgress?.(Math.min(i + batchSize, texts.length), texts.length);
|
|
164
|
+
|
|
165
|
+
// Rate limiting (100ms between batches)
|
|
166
|
+
if (i + batchSize < texts.length) {
|
|
167
|
+
await this.delay(100);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
return results;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* 벡터를 PostgreSQL vector 타입 문자열로 변환
|
|
176
|
+
*/
|
|
177
|
+
static toVectorString(embedding: number[]): string {
|
|
178
|
+
return `[${embedding.join(",")}]`;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* 임베딩 provider의 차원 수 반환
|
|
183
|
+
*/
|
|
184
|
+
getDimensions(provider: EmbeddingProvider): number {
|
|
185
|
+
return provider === "voyage"
|
|
186
|
+
? this.config.voyage.dimensions
|
|
187
|
+
: this.config.openai.dimensions;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
private delay(ms: number): Promise<void> {
|
|
191
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
192
|
+
}
|
|
193
|
+
}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* pgvector 통합을 위한 타입 정의
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
/** 임베딩 제공자 */
|
|
6
|
+
export type EmbeddingProvider = "voyage" | "openai";
|
|
7
|
+
|
|
8
|
+
/** 입력 타입 (Voyage AI 전용 - 비대칭 임베딩) */
|
|
9
|
+
export type VectorInputType = "document" | "query";
|
|
10
|
+
|
|
11
|
+
/** 임베딩 결과 */
|
|
12
|
+
export interface EmbeddingResult {
|
|
13
|
+
embedding: number[];
|
|
14
|
+
model: string;
|
|
15
|
+
tokenCount: number;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** 청크 정보 */
|
|
19
|
+
export interface Chunk {
|
|
20
|
+
index: number;
|
|
21
|
+
text: string;
|
|
22
|
+
startOffset: number;
|
|
23
|
+
endOffset: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** 벡터 검색 결과 */
|
|
27
|
+
export interface VectorSearchResult<T = Record<string, unknown>> {
|
|
28
|
+
id: number | string;
|
|
29
|
+
similarity: number;
|
|
30
|
+
data: T;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** 하이브리드 검색 결과 (Vector + FTS) */
|
|
34
|
+
export interface HybridSearchResult<T = Record<string, unknown>>
|
|
35
|
+
extends VectorSearchResult<T> {
|
|
36
|
+
vectorScore?: number;
|
|
37
|
+
ftsScore?: number;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** 벤치마크 결과 */
|
|
41
|
+
export interface BenchmarkResult {
|
|
42
|
+
provider: EmbeddingProvider;
|
|
43
|
+
embedTime: number;
|
|
44
|
+
searchTime: number;
|
|
45
|
+
results: VectorSearchResult[];
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Voyage AI 설정 */
|
|
49
|
+
export interface VoyageConfig {
|
|
50
|
+
apiKey: string;
|
|
51
|
+
baseUrl: string;
|
|
52
|
+
model: string;
|
|
53
|
+
dimensions: number;
|
|
54
|
+
maxTokens: number;
|
|
55
|
+
batchSize: number;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** OpenAI 설정 */
|
|
59
|
+
export interface OpenAIConfig {
|
|
60
|
+
apiKey: string;
|
|
61
|
+
baseUrl: string;
|
|
62
|
+
model: string;
|
|
63
|
+
dimensions: number;
|
|
64
|
+
maxTokens: number;
|
|
65
|
+
batchSize: number;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** 청킹 설정 */
|
|
69
|
+
export interface ChunkingConfig {
|
|
70
|
+
chunkSize: number;
|
|
71
|
+
chunkOverlap: number;
|
|
72
|
+
minChunkSize: number;
|
|
73
|
+
skipThreshold: number;
|
|
74
|
+
separators: string[];
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** 검색 설정 */
|
|
78
|
+
export interface SearchConfig {
|
|
79
|
+
defaultLimit: number;
|
|
80
|
+
similarityThreshold: number;
|
|
81
|
+
vectorWeight: number;
|
|
82
|
+
ftsWeight: number;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** pgvector 설정 */
|
|
86
|
+
export interface PgvectorConfig {
|
|
87
|
+
iterativeScan: boolean;
|
|
88
|
+
efSearch: number;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** 전체 벡터 설정 */
|
|
92
|
+
export interface VectorConfig {
|
|
93
|
+
voyage: VoyageConfig;
|
|
94
|
+
openai: OpenAIConfig;
|
|
95
|
+
chunking: ChunkingConfig;
|
|
96
|
+
search: SearchConfig;
|
|
97
|
+
pgvector: PgvectorConfig;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** 벡터 검색 옵션 */
|
|
101
|
+
export interface VectorSearchOptions {
|
|
102
|
+
embeddingColumn?: string;
|
|
103
|
+
limit?: number;
|
|
104
|
+
threshold?: number;
|
|
105
|
+
where?: string;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** 하이브리드 검색 옵션 */
|
|
109
|
+
export interface HybridSearchOptions extends VectorSearchOptions {
|
|
110
|
+
vectorWeight?: number;
|
|
111
|
+
ftsWeight?: number;
|
|
112
|
+
ftsColumn?: string;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** 임베딩 저장 항목 */
|
|
116
|
+
export interface EmbeddingItem {
|
|
117
|
+
id: number;
|
|
118
|
+
text: string;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** 진행률 콜백 */
|
|
122
|
+
export type ProgressCallback = (processed: number, total: number) => void;
|
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
import type { Knex } from "knex";
|
|
2
|
+
import pgvector from "pgvector/knex";
|
|
3
|
+
import { DEFAULT_VECTOR_CONFIG } from "./config";
|
|
4
|
+
import { Embedding } from "./embedding";
|
|
5
|
+
import type {
|
|
6
|
+
EmbeddingItem,
|
|
7
|
+
EmbeddingProvider,
|
|
8
|
+
HybridSearchOptions,
|
|
9
|
+
HybridSearchResult,
|
|
10
|
+
ProgressCallback,
|
|
11
|
+
VectorConfig,
|
|
12
|
+
VectorSearchOptions,
|
|
13
|
+
VectorSearchResult,
|
|
14
|
+
} from "./types";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* 벡터 검색
|
|
18
|
+
* pgvector를 활용한 벡터 검색 및 하이브리드 검색 지원
|
|
19
|
+
*/
|
|
20
|
+
export class VectorSearch<T = Record<string, unknown>> {
|
|
21
|
+
private db: Knex;
|
|
22
|
+
private config: VectorConfig;
|
|
23
|
+
private embedding: Embedding;
|
|
24
|
+
private tableName: string;
|
|
25
|
+
|
|
26
|
+
constructor(db: Knex, tableName: string, config: Partial<VectorConfig> = {}) {
|
|
27
|
+
this.db = db;
|
|
28
|
+
this.tableName = tableName;
|
|
29
|
+
this.config = {
|
|
30
|
+
voyage: { ...DEFAULT_VECTOR_CONFIG.voyage, ...config.voyage },
|
|
31
|
+
openai: { ...DEFAULT_VECTOR_CONFIG.openai, ...config.openai },
|
|
32
|
+
chunking: { ...DEFAULT_VECTOR_CONFIG.chunking, ...config.chunking },
|
|
33
|
+
search: { ...DEFAULT_VECTOR_CONFIG.search, ...config.search },
|
|
34
|
+
pgvector: { ...DEFAULT_VECTOR_CONFIG.pgvector, ...config.pgvector },
|
|
35
|
+
};
|
|
36
|
+
this.embedding = new Embedding(config);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* 단일 항목에 임베딩 저장
|
|
41
|
+
*/
|
|
42
|
+
async saveEmbedding(
|
|
43
|
+
id: number,
|
|
44
|
+
text: string,
|
|
45
|
+
provider: EmbeddingProvider,
|
|
46
|
+
embeddingColumn: string = "content_embedding",
|
|
47
|
+
): Promise<void> {
|
|
48
|
+
const { embedding } = await this.embedding.embedOne(text, provider, "document");
|
|
49
|
+
|
|
50
|
+
await this.db(this.tableName)
|
|
51
|
+
.where("id", id)
|
|
52
|
+
.update({
|
|
53
|
+
[embeddingColumn]: pgvector.toSql(embedding),
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* 여러 항목에 임베딩 일괄 저장
|
|
59
|
+
*/
|
|
60
|
+
async saveEmbeddingsBatch(
|
|
61
|
+
items: EmbeddingItem[],
|
|
62
|
+
provider: EmbeddingProvider,
|
|
63
|
+
embeddingColumn: string = "content_embedding",
|
|
64
|
+
onProgress?: ProgressCallback,
|
|
65
|
+
): Promise<void> {
|
|
66
|
+
const texts = items.map((item) => item.text);
|
|
67
|
+
const embeddings = await this.embedding.embedBatch(texts, provider, "document", onProgress);
|
|
68
|
+
|
|
69
|
+
await this.db.transaction(async (trx) => {
|
|
70
|
+
for (let i = 0; i < items.length; i++) {
|
|
71
|
+
await trx(this.tableName)
|
|
72
|
+
.where("id", items[i].id)
|
|
73
|
+
.update({
|
|
74
|
+
[embeddingColumn]: pgvector.toSql(embeddings[i].embedding),
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* 벡터 검색 (코사인 유사도)
|
|
82
|
+
*/
|
|
83
|
+
async search(
|
|
84
|
+
query: string,
|
|
85
|
+
provider: EmbeddingProvider,
|
|
86
|
+
options: VectorSearchOptions = {},
|
|
87
|
+
): Promise<VectorSearchResult<T>[]> {
|
|
88
|
+
const {
|
|
89
|
+
embeddingColumn = "content_embedding",
|
|
90
|
+
limit = this.config.search.defaultLimit,
|
|
91
|
+
threshold = this.config.search.similarityThreshold,
|
|
92
|
+
where,
|
|
93
|
+
} = options;
|
|
94
|
+
|
|
95
|
+
// 쿼리 임베딩 (input_type: 'query' 중요!)
|
|
96
|
+
const { embedding } = await this.embedding.embedOne(query, provider, "query");
|
|
97
|
+
|
|
98
|
+
// pgvector 세션 설정
|
|
99
|
+
if (this.config.pgvector.iterativeScan) {
|
|
100
|
+
await this.db.raw("SET hnsw.iterative_scan = relaxed_order");
|
|
101
|
+
}
|
|
102
|
+
await this.db.raw(`SET hnsw.ef_search = ${this.config.pgvector.efSearch}`);
|
|
103
|
+
|
|
104
|
+
// 코사인 유사도 = 1 - 코사인 거리
|
|
105
|
+
const vectorStr = pgvector.toSql(embedding);
|
|
106
|
+
let queryBuilder = this.db(this.tableName)
|
|
107
|
+
.select("*")
|
|
108
|
+
.select(this.db.raw(`1 - (${embeddingColumn} <=> ?::vector) AS similarity`, [vectorStr]))
|
|
109
|
+
.whereNotNull(embeddingColumn)
|
|
110
|
+
.orderByRaw(`${embeddingColumn} <=> ?::vector`, [vectorStr])
|
|
111
|
+
.limit(limit);
|
|
112
|
+
|
|
113
|
+
if (where) {
|
|
114
|
+
queryBuilder = queryBuilder.whereRaw(where);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
const rows = await queryBuilder;
|
|
118
|
+
|
|
119
|
+
return rows
|
|
120
|
+
.filter((row: { similarity: number }) => row.similarity >= threshold)
|
|
121
|
+
.map((row: T & { similarity: number }) => ({
|
|
122
|
+
id: (row as unknown as { id: number }).id,
|
|
123
|
+
similarity: parseFloat(String(row.similarity)),
|
|
124
|
+
data: row as T,
|
|
125
|
+
}));
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* 하이브리드 검색 (Vector + FTS)
|
|
130
|
+
*/
|
|
131
|
+
async hybridSearch(
|
|
132
|
+
query: string,
|
|
133
|
+
provider: EmbeddingProvider,
|
|
134
|
+
options: HybridSearchOptions = {},
|
|
135
|
+
): Promise<HybridSearchResult<T>[]> {
|
|
136
|
+
const {
|
|
137
|
+
embeddingColumn = "content_embedding",
|
|
138
|
+
ftsColumn = "content_tsv",
|
|
139
|
+
limit = this.config.search.defaultLimit,
|
|
140
|
+
vectorWeight = this.config.search.vectorWeight,
|
|
141
|
+
ftsWeight = this.config.search.ftsWeight,
|
|
142
|
+
} = options;
|
|
143
|
+
|
|
144
|
+
const { embedding } = await this.embedding.embedOne(query, provider, "query");
|
|
145
|
+
const vectorStr = pgvector.toSql(embedding);
|
|
146
|
+
|
|
147
|
+
// pgvector 세션 설정
|
|
148
|
+
if (this.config.pgvector.iterativeScan) {
|
|
149
|
+
await this.db.raw("SET hnsw.iterative_scan = relaxed_order");
|
|
150
|
+
}
|
|
151
|
+
await this.db.raw(`SET hnsw.ef_search = ${this.config.pgvector.efSearch}`);
|
|
152
|
+
|
|
153
|
+
const sql = `
|
|
154
|
+
WITH vector_search AS (
|
|
155
|
+
SELECT
|
|
156
|
+
id,
|
|
157
|
+
ROW_NUMBER() OVER (ORDER BY ${embeddingColumn} <=> ?::vector) AS rank
|
|
158
|
+
FROM ${this.tableName}
|
|
159
|
+
WHERE ${embeddingColumn} IS NOT NULL
|
|
160
|
+
ORDER BY ${embeddingColumn} <=> ?::vector
|
|
161
|
+
LIMIT 50
|
|
162
|
+
),
|
|
163
|
+
fts_search AS (
|
|
164
|
+
SELECT
|
|
165
|
+
id,
|
|
166
|
+
ROW_NUMBER() OVER (ORDER BY ts_rank(${ftsColumn}, query) DESC) AS rank
|
|
167
|
+
FROM ${this.tableName}, plainto_tsquery('simple', ?) query
|
|
168
|
+
WHERE ${ftsColumn} @@ query
|
|
169
|
+
LIMIT 50
|
|
170
|
+
),
|
|
171
|
+
combined AS (
|
|
172
|
+
SELECT
|
|
173
|
+
COALESCE(v.id, f.id) AS id,
|
|
174
|
+
COALESCE(1.0 / (60 + v.rank), 0) AS vector_score,
|
|
175
|
+
COALESCE(1.0 / (60 + f.rank), 0) AS fts_score
|
|
176
|
+
FROM vector_search v
|
|
177
|
+
FULL OUTER JOIN fts_search f ON v.id = f.id
|
|
178
|
+
)
|
|
179
|
+
SELECT
|
|
180
|
+
t.*,
|
|
181
|
+
c.vector_score,
|
|
182
|
+
c.fts_score,
|
|
183
|
+
(c.vector_score * ? + c.fts_score * ?) AS similarity
|
|
184
|
+
FROM combined c
|
|
185
|
+
JOIN ${this.tableName} t ON c.id = t.id
|
|
186
|
+
ORDER BY similarity DESC
|
|
187
|
+
LIMIT ?
|
|
188
|
+
`;
|
|
189
|
+
|
|
190
|
+
const { rows } = await this.db.raw(sql, [
|
|
191
|
+
vectorStr,
|
|
192
|
+
vectorStr,
|
|
193
|
+
query,
|
|
194
|
+
vectorWeight,
|
|
195
|
+
ftsWeight,
|
|
196
|
+
limit,
|
|
197
|
+
]);
|
|
198
|
+
|
|
199
|
+
return rows.map(
|
|
200
|
+
(
|
|
201
|
+
row: T & {
|
|
202
|
+
similarity: number;
|
|
203
|
+
vector_score: number;
|
|
204
|
+
fts_score: number;
|
|
205
|
+
},
|
|
206
|
+
) => ({
|
|
207
|
+
id: (row as unknown as { id: number }).id,
|
|
208
|
+
similarity: parseFloat(String(row.similarity)),
|
|
209
|
+
vectorScore: parseFloat(String(row.vector_score)),
|
|
210
|
+
ftsScore: parseFloat(String(row.fts_score)),
|
|
211
|
+
data: row as T,
|
|
212
|
+
}),
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* 임베딩 현황 조회
|
|
218
|
+
*/
|
|
219
|
+
async getEmbeddingStatus(embeddingColumn: string = "content_embedding"): Promise<{
|
|
220
|
+
total: number;
|
|
221
|
+
withEmbedding: number;
|
|
222
|
+
withoutEmbedding: number;
|
|
223
|
+
}> {
|
|
224
|
+
const result = await this.db(this.tableName)
|
|
225
|
+
.count("* as total")
|
|
226
|
+
.count(`${embeddingColumn} as with_embedding`)
|
|
227
|
+
.first();
|
|
228
|
+
|
|
229
|
+
const total = parseInt(String(result?.total ?? 0), 10);
|
|
230
|
+
const withEmbedding = parseInt(String(result?.with_embedding ?? 0), 10);
|
|
231
|
+
|
|
232
|
+
return {
|
|
233
|
+
total,
|
|
234
|
+
withEmbedding,
|
|
235
|
+
withoutEmbedding: total - withEmbedding,
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* 임베딩이 없는 항목 ID 조회
|
|
241
|
+
*/
|
|
242
|
+
async getItemsWithoutEmbedding(
|
|
243
|
+
embeddingColumn: string = "content_embedding",
|
|
244
|
+
limit: number = 100,
|
|
245
|
+
): Promise<number[]> {
|
|
246
|
+
const rows = await this.db(this.tableName)
|
|
247
|
+
.select("id")
|
|
248
|
+
.whereNull(embeddingColumn)
|
|
249
|
+
.orderBy("id")
|
|
250
|
+
.limit(limit);
|
|
251
|
+
|
|
252
|
+
return rows.map((row: { id: number }) => row.id);
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Embedding 인스턴스 반환 (고급 사용)
|
|
257
|
+
*/
|
|
258
|
+
getEmbedding(): Embedding {
|
|
259
|
+
return this.embedding;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import { type EntityNamesRecord } from "../../entity/entity-manager";
|
|
2
|
-
import type { TemplateOptions } from "../../types/types";
|
|
3
|
-
import { Template } from "../template";
|
|
4
|
-
export declare class Template__view_enums_buttonset extends Template {
|
|
5
|
-
constructor();
|
|
6
|
-
getTargetAndPath(names: EntityNamesRecord, componentId: string): {
|
|
7
|
-
target: string;
|
|
8
|
-
path: string;
|
|
9
|
-
};
|
|
10
|
-
render({ entityId, enumId }: TemplateOptions["view_enums_buttonset"]): {
|
|
11
|
-
body: string;
|
|
12
|
-
importKeys: never[];
|
|
13
|
-
target: string;
|
|
14
|
-
path: string;
|
|
15
|
-
};
|
|
16
|
-
}
|
|
17
|
-
//# sourceMappingURL=view_enums_buttonset.template.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"view_enums_buttonset.template.d.ts","sourceRoot":"","sources":["../../../src/template/implementations/view_enums_buttonset.template.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,KAAK,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AACpF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,qBAAa,8BAA+B,SAAQ,QAAQ;;IAK1D,gBAAgB,CAAC,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM;;;;IAO9D,MAAM,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,eAAe,CAAC,sBAAsB,CAAC;;;;;;CAiBrE"}
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
import { EntityManager } from "../../entity/entity-manager.js";
|
|
2
|
-
import { Template } from "../template.js";
|
|
3
|
-
export class Template__view_enums_buttonset extends Template {
|
|
4
|
-
constructor(){
|
|
5
|
-
super("view_enums_buttonset");
|
|
6
|
-
}
|
|
7
|
-
getTargetAndPath(names, componentId) {
|
|
8
|
-
return {
|
|
9
|
-
target: "web/src/components",
|
|
10
|
-
path: `${names.fs}/${componentId}ButtonSet.tsx`
|
|
11
|
-
};
|
|
12
|
-
}
|
|
13
|
-
render({ entityId, enumId }) {
|
|
14
|
-
const names = EntityManager.getNamesFromId(entityId);
|
|
15
|
-
return {
|
|
16
|
-
...this.getTargetAndPath(names, enumId),
|
|
17
|
-
body: `
|
|
18
|
-
/*
|
|
19
|
-
view_enums_buttonset
|
|
20
|
-
${JSON.stringify({
|
|
21
|
-
key: this.key,
|
|
22
|
-
options: entityId
|
|
23
|
-
})}
|
|
24
|
-
*/
|
|
25
|
-
`.trim(),
|
|
26
|
-
importKeys: []
|
|
27
|
-
};
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbIi4uLy4uLy4uL3NyYy90ZW1wbGF0ZS9pbXBsZW1lbnRhdGlvbnMvdmlld19lbnVtc19idXR0b25zZXQudGVtcGxhdGUudHMiXSwic291cmNlc0NvbnRlbnQiOlsiaW1wb3J0IHsgRW50aXR5TWFuYWdlciwgdHlwZSBFbnRpdHlOYW1lc1JlY29yZCB9IGZyb20gXCIuLi8uLi9lbnRpdHkvZW50aXR5LW1hbmFnZXJcIjtcbmltcG9ydCB0eXBlIHsgVGVtcGxhdGVPcHRpb25zIH0gZnJvbSBcIi4uLy4uL3R5cGVzL3R5cGVzXCI7XG5pbXBvcnQgeyBUZW1wbGF0ZSB9IGZyb20gXCIuLi90ZW1wbGF0ZVwiO1xuXG5leHBvcnQgY2xhc3MgVGVtcGxhdGVfX3ZpZXdfZW51bXNfYnV0dG9uc2V0IGV4dGVuZHMgVGVtcGxhdGUge1xuICBjb25zdHJ1Y3RvcigpIHtcbiAgICBzdXBlcihcInZpZXdfZW51bXNfYnV0dG9uc2V0XCIpO1xuICB9XG5cbiAgZ2V0VGFyZ2V0QW5kUGF0aChuYW1lczogRW50aXR5TmFtZXNSZWNvcmQsIGNvbXBvbmVudElkOiBzdHJpbmcpIHtcbiAgICByZXR1cm4ge1xuICAgICAgdGFyZ2V0OiBcIndlYi9zcmMvY29tcG9uZW50c1wiLFxuICAgICAgcGF0aDogYCR7bmFtZXMuZnN9LyR7Y29tcG9uZW50SWR9QnV0dG9uU2V0LnRzeGAsXG4gICAgfTtcbiAgfVxuXG4gIHJlbmRlcih7IGVudGl0eUlkLCBlbnVtSWQgfTogVGVtcGxhdGVPcHRpb25zW1widmlld19lbnVtc19idXR0b25zZXRcIl0pIHtcbiAgICBjb25zdCBuYW1lcyA9IEVudGl0eU1hbmFnZXIuZ2V0TmFtZXNGcm9tSWQoZW50aXR5SWQpO1xuXG4gICAgcmV0dXJuIHtcbiAgICAgIC4uLnRoaXMuZ2V0VGFyZ2V0QW5kUGF0aChuYW1lcywgZW51bUlkKSxcbiAgICAgIGJvZHk6IGBcbi8qXG52aWV3X2VudW1zX2J1dHRvbnNldFxuJHtKU09OLnN0cmluZ2lmeSh7XG4gIGtleTogdGhpcy5rZXksXG4gIG9wdGlvbnM6IGVudGl0eUlkLFxufSl9XG4qL1xuICAgICAgYC50cmltKCksXG4gICAgICBpbXBvcnRLZXlzOiBbXSxcbiAgICB9O1xuICB9XG59XG4iXSwibmFtZXMiOlsiRW50aXR5TWFuYWdlciIsIlRlbXBsYXRlIiwiVGVtcGxhdGVfX3ZpZXdfZW51bXNfYnV0dG9uc2V0IiwiZ2V0VGFyZ2V0QW5kUGF0aCIsIm5hbWVzIiwiY29tcG9uZW50SWQiLCJ0YXJnZXQiLCJwYXRoIiwiZnMiLCJyZW5kZXIiLCJlbnRpdHlJZCIsImVudW1JZCIsImdldE5hbWVzRnJvbUlkIiwiYm9keSIsIkpTT04iLCJzdHJpbmdpZnkiLCJrZXkiLCJvcHRpb25zIiwidHJpbSIsImltcG9ydEtleXMiXSwibWFwcGluZ3MiOiJBQUFBLFNBQVNBLGFBQWEsUUFBZ0MsaUNBQThCO0FBRXBGLFNBQVNDLFFBQVEsUUFBUSxpQkFBYztBQUV2QyxPQUFPLE1BQU1DLHVDQUF1Q0Q7SUFDbEQsYUFBYztRQUNaLEtBQUssQ0FBQztJQUNSO0lBRUFFLGlCQUFpQkMsS0FBd0IsRUFBRUMsV0FBbUIsRUFBRTtRQUM5RCxPQUFPO1lBQ0xDLFFBQVE7WUFDUkMsTUFBTSxHQUFHSCxNQUFNSSxFQUFFLENBQUMsQ0FBQyxFQUFFSCxZQUFZLGFBQWEsQ0FBQztRQUNqRDtJQUNGO0lBRUFJLE9BQU8sRUFBRUMsUUFBUSxFQUFFQyxNQUFNLEVBQTJDLEVBQUU7UUFDcEUsTUFBTVAsUUFBUUosY0FBY1ksY0FBYyxDQUFDRjtRQUUzQyxPQUFPO1lBQ0wsR0FBRyxJQUFJLENBQUNQLGdCQUFnQixDQUFDQyxPQUFPTyxPQUFPO1lBQ3ZDRSxNQUFNLENBQUM7OztBQUdiLEVBQUVDLEtBQUtDLFNBQVMsQ0FBQztnQkFDZkMsS0FBSyxJQUFJLENBQUNBLEdBQUc7Z0JBQ2JDLFNBQVNQO1lBQ1gsR0FBRzs7TUFFRyxDQUFDLENBQUNRLElBQUk7WUFDTkMsWUFBWSxFQUFFO1FBQ2hCO0lBQ0Y7QUFDRiJ9
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import { type EntityNamesRecord } from "../../entity/entity-manager";
|
|
2
|
-
import type { TemplateOptions } from "../../types/types";
|
|
3
|
-
import { Template } from "../template";
|
|
4
|
-
export declare class Template__view_list_columns extends Template {
|
|
5
|
-
constructor();
|
|
6
|
-
getTargetAndPath(names: EntityNamesRecord): {
|
|
7
|
-
target: string;
|
|
8
|
-
path: string;
|
|
9
|
-
};
|
|
10
|
-
render({ entityId, columns, columnImports }: TemplateOptions["view_list_columns"]): {
|
|
11
|
-
body: string;
|
|
12
|
-
importKeys: never[];
|
|
13
|
-
target: string;
|
|
14
|
-
path: string;
|
|
15
|
-
};
|
|
16
|
-
}
|
|
17
|
-
//# sourceMappingURL=view_list_columns.template.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"view_list_columns.template.d.ts","sourceRoot":"","sources":["../../../src/template/implementations/view_list_columns.template.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,KAAK,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AACpF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,qBAAa,2BAA4B,SAAQ,QAAQ;;IAKvD,gBAAgB,CAAC,KAAK,EAAE,iBAAiB;;;;IAQzC,MAAM,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE,EAAE,eAAe,CAAC,mBAAmB,CAAC;;;;;;CAmClF"}
|