sonamu 0.7.8 → 0.7.10

package/src/migration/postgresql-schema-reader.ts

@@ -148,6 +148,7 @@ class PostgreSQLSchemaReaderClass {
           sortOrder: idx.sort_order,
         })),
         nullsNotDistinct: firstIndex.nulls_not_distinct,
+        using: firstIndex.index_type as "btree" | "hash" | "gin" | "gist" | undefined,
       };
     });
 

package/src/template/implementations/generated.template.ts

@@ -4,7 +4,7 @@ import { Sonamu } from "../../api";
 import type { Entity } from "../../entity/entity";
 import { EntityManager } from "../../entity/entity-manager";
 import { Naite } from "../../naite/naite";
-import { type EntityPropNode, isVirtualProp } from "../../types/types";
+import { type EntityIndex, type EntityPropNode, isVirtualProp } from "../../types/types";
 import { nonNullable } from "../../utils/utils";
 import { Template } from "../template";
 import { propNodeToZodTypeDef, zodTypeToZodCode } from "../zod-converter";
@@ -168,9 +168,8 @@ export class Template__generated extends Template {
     })();
 
     // fulltext index에 포함된 컬럼들 추출
-    const fulltextColumns = unique(
-      entity.indexes.filter((index) => index.type === "fulltext").flatMap((index) => index.columns),
-    );
+    // TODO: GIN/GiST 인덱스 생성된 컬럼 추출
+    const fulltextColumns: EntityIndex["columns"][] = [];
 
     // virtual props
     const virtualProps = entity.props

package/src/types/types.ts

@@ -113,6 +113,9 @@ export type VectorArrayProp = CommonProp & {
   type: "vector[]";
   dimensions: number;
 };
+export type TsVectorProp = CommonProp & {
+  type: "tsvector";
+};
 export type RelationType = "HasMany" | "BelongsToOne" | "ManyToMany" | "OneToOne";
 export type RelationOn = "CASCADE" | "SET NULL" | "NO ACTION" | "SET DEFAULT" | "RESTRICT";
 type _RelationProp = {
@@ -185,6 +188,7 @@ export type EntityProp =
   | VirtualProp
   | VectorProp
   | VectorArrayProp
+  | TsVectorProp
   | RelationProp;
 
 /**
@@ -220,10 +224,10 @@ type EntityIndexColumn = {
   vectorOps?: VectorOps;
 };
 export type EntityIndex = {
-  type: "index" | "unique" | "fulltext" | "hnsw" | "ivfflat";
+  type: "index" | "unique" | "hnsw" | "ivfflat";
   columns: EntityIndexColumn[];
   name: string;
-  parser?: "built-in" | "ngram";
+  using?: "btree" | "hash" | "gin" | "gist";
   nullsNotDistinct?: boolean; // unique index only
   /**
    * HNSW (Hierarchical Navigable Small World) 인덱스: 각 노드의 최대 연결 수
@@ -547,10 +551,10 @@ export type MigrationColumn = {
   generated?: GeneratedColumn;
 };
 export type MigrationIndex = {
-  type: "unique" | "index" | "fulltext" | "hnsw" | "ivfflat";
+  type: "unique" | "index" | "hnsw" | "ivfflat";
   columns: EntityIndexColumn[];
   name: string;
-  parser?: "built-in" | "ngram";
+  using?: "btree" | "hash" | "gin" | "gist";
   nullsNotDistinct?: boolean;
   /** HNSW (Hierarchical Navigable Small World): 각 노드의 최대 연결 수 */
   m?: number;
@@ -842,6 +846,7 @@ const BasePropFieldsWithoutAdditional = z
       z.literal("date[]"),
       z.literal("uuid"),
       z.literal("uuid[]"),
+      z.literal("tsvector"),
     ]),
   })
   .strict();
@@ -1038,6 +1043,7 @@ const NormalPropTypes = [
   "virtual",
   "vector",
   "vector[]",
+  "tsvector",
 ] as const;
 
 // VIRTUAL Generated Column에서 사용 불가능한 타입들
@@ -1131,10 +1137,10 @@ const EntityIndexColumnSchema = z.object({
 // EntityIndex 스키마 정의
 const EntityIndexSchema = z
   .object({
-    type: z.enum(["index", "unique", "fulltext", "hnsw", "ivfflat"]),
+    type: z.enum(["index", "unique", "hnsw", "ivfflat"]),
     columns: z.array(EntityIndexColumnSchema),
     name: z.string().min(1).max(63),
-    parser: z.enum(["built-in", "ngram"]).optional(),
+    using: z.enum(["btree", "hash", "gin", "gist"]).optional(),
     nullsNotDistinct: z.boolean().optional(),
     m: z.number().optional(),
     efConstruction: z.number().optional(),

package/src/vector/config.ts

@@ -13,7 +13,7 @@ export const DEFAULT_VECTOR_CONFIG: VectorConfig = {
     model: "voyage-3",
     dimensions: 1024,
     maxTokens: 32000,
-    batchSize: 100,
+    batchSize: 128,
   },
 
   // OpenAI 설정
@@ -55,9 +55,7 @@ export const DEFAULT_VECTOR_CONFIG: VectorConfig = {
  * 설정 생성 헬퍼 함수
  * 부분 설정만 제공하면 나머지는 기본값 사용
  */
-export function createVectorConfig(
-  overrides: Partial<VectorConfig> = {}
-): VectorConfig {
+export function createVectorConfig(overrides: Partial<VectorConfig> = {}): VectorConfig {
   return {
     voyage: { ...DEFAULT_VECTOR_CONFIG.voyage, ...overrides.voyage },
     openai: { ...DEFAULT_VECTOR_CONFIG.openai, ...overrides.openai },

package/src/vector/embedding.ts

@@ -1,3 +1,6 @@
+import { createOpenAI, type OpenAIProvider } from "@ai-sdk/openai";
+import { type EmbeddingModel, embedMany } from "ai";
+import { VoyageAIClient } from "voyageai";
 import { Sonamu } from "../api/sonamu";
 import { DEFAULT_VECTOR_CONFIG } from "./config";
 import type {
@@ -10,7 +13,7 @@ import type {
 
 /**
  * 임베딩 클라이언트
- * Voyage AI와 OpenAI 임베딩을 통합 지원
+ * Voyage AI와 OpenAI 임베딩을 SDK 방식으로 통합 지원
  */
 export class Embedding {
   private config: VectorConfig;
@@ -25,22 +28,64 @@ export class Embedding {
     };
   }
 
+  /**
+   * Voyage AI 클라이언트 초기화
+   */
+  private getVoyageClient(): VoyageAIClient {
+    const apiKey = Sonamu.secrets?.voyage_api_key ?? process.env.VOYAGE_API_KEY;
+    if (!apiKey) {
+      throw new Error("VOYAGE_API_KEY가 설정되지 않았습니다. 환경변수를 확인하세요.");
+    }
+    return new VoyageAIClient({ apiKey });
+  }
+
+  /**
+   * OpenAI provider 생성
+   */
+  private getOpenAIProvider(): OpenAIProvider {
+    const apiKey = Sonamu.secrets?.openai_api_key ?? process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+      throw new Error("OPENAI_API_KEY가 설정되지 않았습니다. 환경변수를 확인하세요.");
+    }
+    return createOpenAI({ apiKey });
+  }
+
   /**
    * 텍스트 임베딩 생성
-   * @param texts - 임베딩할 텍스트 배열
+   * @param texts - 임베딩할 텍스트 배열 (batchSize이상 시 자동 분할)
    * @param provider - 'voyage' | 'openai'
    * @param inputType - 'document' | 'query' (Voyage AI만 해당)
+   * @param onProgress - 진행률 콜백
    */
   async embed(
     texts: string[],
     provider: EmbeddingProvider,
-    inputType: VectorInputType = "document"
+    inputType: VectorInputType = "document",
+    onProgress?: ProgressCallback,
   ): Promise<EmbeddingResult[]> {
-    if (provider === "voyage") {
-      return this.embedVoyage(texts, inputType);
-    } else {
-      return this.embedOpenAI(texts);
+    const maxBatchSize =
+      provider === "voyage" ? this.config.voyage.batchSize : this.config.openai.batchSize;
+
+    // batchSize이하면 바로 호출
+    if (texts.length <= maxBatchSize) {
+      return provider === "voyage"
+        ? await this.embedVoyage(texts, inputType)
+        : await this.embedOpenAI(texts);
     }
+
+    // batchSize이상이면 자동으로 나눠서 처리
+    const batches = Array.from({ length: Math.ceil(texts.length / maxBatchSize) }, (_, i) =>
+      texts.slice(i * maxBatchSize, (i + 1) * maxBatchSize),
+    );
+
+    const results = await Promise.all(
+      batches.map((batch) =>
+        provider === "voyage" ? this.embedVoyage(batch, inputType) : this.embedOpenAI(batch),
+      ),
+    );
+
+    onProgress?.(texts.length, texts.length);
+    return results.flat();
   }
 
   /**
@@ -49,7 +94,7 @@ export class Embedding {
   async embedOne(
     text: string,
     provider: EmbeddingProvider,
-    inputType: VectorInputType = "document"
+    inputType: VectorInputType = "document",
   ): Promise<EmbeddingResult> {
     const results = await this.embed([text], provider, inputType);
     return results[0];
@@ -60,42 +105,24 @@ export class Embedding {
    */
   private async embedVoyage(
     texts: string[],
-    inputType: VectorInputType
+    inputType: VectorInputType,
   ): Promise<EmbeddingResult[]> {
+    const client = this.getVoyageClient();
     const voyageConfig = this.config.voyage;
 
-    // config에서 설정된 apiKey 우선, 없으면 Sonamu.secrets에서 로드
-    const apiKey = voyageConfig.apiKey || Sonamu.secrets?.voyage_api_key;
-    if (!apiKey) {
-      throw new Error(
-        "VOYAGE_API_KEY가 설정되지 않았습니다. 환경변수를 확인하세요."
-      );
-    }
-
-    const response = await fetch(voyageConfig.baseUrl, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        Authorization: `Bearer ${apiKey}`,
-      },
-      body: JSON.stringify({
-        input: texts,
-        model: voyageConfig.model,
-        input_type: inputType,
-      }),
+    const response = await client.embed({
+      input: texts,
+      model: voyageConfig.model,
+      inputType: inputType,
     });
-
-    if (!response.ok) {
-      const error = await response.text();
-      throw new Error(`Voyage API error: ${response.status} - ${error}`);
+    if (!response.data) {
+      throw new Error("Voyage API: 응답 데이터가 없습니다.");
     }
 
-    const data = await response.json();
-
-    return data.data.map((item: { embedding: number[] }) => ({
-      embedding: item.embedding,
+    return response.data.map((item) => ({
+      embedding: item.embedding ?? [],
       model: voyageConfig.model,
-      tokenCount: data.usage?.total_tokens || 0,
+      tokenCount: response.usage?.totalTokens ?? 0,
     }));
   }
 
@@ -103,74 +130,22 @@ export class Embedding {
    * OpenAI 임베딩
    */
   private async embedOpenAI(texts: string[]): Promise<EmbeddingResult[]> {
+    const openai = this.getOpenAIProvider();
     const openaiConfig = this.config.openai;
+    const model = openai.embeddingModel(openaiConfig.model);
 
-    // config에서 설정된 apiKey 우선, 없으면 Sonamu.secrets에서 로드
-    const apiKey = openaiConfig.apiKey || Sonamu.secrets?.openai_api_key;
-    if (!apiKey) {
-      throw new Error(
-        "OPENAI_API_KEY가 설정되지 않았습니다. 환경변수를 확인하세요."
-      );
-    }
-
-    const response = await fetch(openaiConfig.baseUrl, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        Authorization: `Bearer ${apiKey}`,
-      },
-      body: JSON.stringify({
-        input: texts,
-        model: openaiConfig.model,
-      }),
+    const { embeddings, usage } = await embedMany({
+      model: model as EmbeddingModel,
+      values: texts,
     });
 
-    if (!response.ok) {
-      const error = await response.text();
-      throw new Error(`OpenAI API error: ${response.status} - ${error}`);
-    }
-
-    const data = await response.json();
-
-    return data.data.map((item: { embedding: number[] }) => ({
-      embedding: item.embedding,
+    return embeddings.map((embedding) => ({
+      embedding,
       model: openaiConfig.model,
-      tokenCount: data.usage?.total_tokens || 0,
+      tokenCount: usage?.tokens ?? 0,
     }));
   }
 
-  /**
-   * 배치 임베딩 (대량 처리)
-   */
-  async embedBatch(
-    texts: string[],
-    provider: EmbeddingProvider,
-    inputType: VectorInputType = "document",
-    onProgress?: ProgressCallback
-  ): Promise<EmbeddingResult[]> {
-    const batchSize =
-      provider === "voyage"
-        ? this.config.voyage.batchSize
-        : this.config.openai.batchSize;
-
-    const results: EmbeddingResult[] = [];
-
-    for (let i = 0; i < texts.length; i += batchSize) {
-      const batch = texts.slice(i, i + batchSize);
-      const batchResults = await this.embed(batch, provider, inputType);
-      results.push(...batchResults);
-
-      onProgress?.(Math.min(i + batchSize, texts.length), texts.length);
-
-      // Rate limiting (100ms between batches)
-      if (i + batchSize < texts.length) {
-        await this.delay(100);
-      }
-    }
-
-    return results;
-  }
-
   /**
    * 벡터를 PostgreSQL vector 타입 문자열로 변환
    */
@@ -182,12 +157,6 @@ export class Embedding {
    * 임베딩 provider의 차원 수 반환
    */
   getDimensions(provider: EmbeddingProvider): number {
-    return provider === "voyage"
-      ? this.config.voyage.dimensions
-      : this.config.openai.dimensions;
-  }
-
-  private delay(ms: number): Promise<void> {
-    return new Promise((resolve) => setTimeout(resolve, ms));
+    return provider === "voyage" ? this.config.voyage.dimensions : this.config.openai.dimensions;
   }
 }

package/src/vector/vector-search.ts

@@ -64,7 +64,7 @@ export class VectorSearch<T = Record<string, unknown>> {
     onProgress?: ProgressCallback,
   ): Promise<void> {
     const texts = items.map((item) => item.text);
-    const embeddings = await this.embedding.embedBatch(texts, provider, "document", onProgress);
+    const embeddings = await this.embedding.embed(texts, provider, "document", onProgress);
 
     await this.db.transaction(async (trx) => {
       for (let i = 0; i < items.length; i++) {