sonamu 0.7.3 → 0.7.5

package/src/vector/chunking.ts

@@ -0,0 +1,115 @@
+import { DEFAULT_VECTOR_CONFIG } from "./config";
+import type { Chunk, ChunkingConfig } from "./types";
+
+/**
+ * 텍스트 청킹
+ * - 현재 가이드에서는 400토큰 이하만 저장하므로 기본적으로 사용하지 않음
+ * - 추후 긴 문서 처리 시 사용
+ */
+export class Chunking {
+  private config: ChunkingConfig;
+
+  constructor(config: Partial<ChunkingConfig> = {}) {
+    this.config = { ...DEFAULT_VECTOR_CONFIG.chunking, ...config };
+  }
+
+  /**
+   * 텍스트를 청크로 분할
+   */
+  chunk(text: string): Chunk[] {
+    if (text.length < this.config.skipThreshold) {
+      return [
+        {
+          index: 0,
+          text: text.trim(),
+          startOffset: 0,
+          endOffset: text.length,
+        },
+      ];
+    }
+
+    const chunks: Chunk[] = [];
+    let currentPosition = 0;
+
+    while (currentPosition < text.length) {
+      const remainingText = text.slice(currentPosition);
+      const { chunk, length } = this.extractChunk(remainingText);
+
+      if (chunk.trim().length >= this.config.minChunkSize) {
+        chunks.push({
+          index: chunks.length,
+          text: chunk.trim(),
+          startOffset: currentPosition,
+          endOffset: currentPosition + length,
+        });
+      }
+
+      const advance = Math.max(length - this.config.chunkOverlap, this.config.minChunkSize);
+      currentPosition += advance;
+    }
+
+    return chunks;
+  }
+
+  /**
+   * 청킹이 필요한지 확인
+   */
+  needsChunking(text: string): boolean {
+    return text.length > this.config.chunkSize;
+  }
+
+  /**
+   * 예상 청크 수 계산
+   */
+  estimateChunkCount(text: string): number {
+    if (text.length <= this.config.chunkSize) {
+      return 1;
+    }
+    const effectiveChunkSize = this.config.chunkSize - this.config.chunkOverlap;
+    return Math.ceil(text.length / effectiveChunkSize);
+  }
+
+  private extractChunk(text: string): { chunk: string; length: number } {
+    if (text.length <= this.config.chunkSize) {
+      return { chunk: text, length: text.length };
+    }
+
+    for (const separator of this.config.separators) {
+      const result = this.splitBySeparator(text, separator);
+      if (result) return result;
+    }
+
+    return {
+      chunk: text.slice(0, this.config.chunkSize),
+      length: this.config.chunkSize,
+    };
+  }
+
+  private splitBySeparator(
+    text: string,
+    separator: string,
+  ): { chunk: string; length: number } | null {
+    const searchRange = text.slice(0, this.config.chunkSize + 100);
+
+    let lastIndex = -1;
+    let index = 0;
+
+    while (true) {
+      index = searchRange.indexOf(separator, index);
+      if (index === -1) break;
+      if (index <= this.config.chunkSize) {
+        lastIndex = index + separator.length;
+      }
+      index++;
+    }
+
+    if (lastIndex > this.config.minChunkSize) {
+      return {
+        chunk: text.slice(0, lastIndex),
+        length: lastIndex,
+      };
+    }
+
+    return null;
+  }
+}

package/src/vector/config.ts

@@ -0,0 +1,68 @@
+import type { VectorConfig } from "./types";
+
+/**
+ * 벡터 검색 기본 설정
+ * 사용자는 이 설정을 override하여 커스터마이즈할 수 있음
+ */
+export const DEFAULT_VECTOR_CONFIG: VectorConfig = {
+  // Voyage AI 설정
+  // apiKey는 Sonamu.secrets에서 로드되므로 여기서는 빈 문자열
+  voyage: {
+    apiKey: "",
+    baseUrl: "https://api.voyageai.com/v1/embeddings",
+    model: "voyage-3",
+    dimensions: 1024,
+    maxTokens: 32000,
+    batchSize: 100,
+  },
+
+  // OpenAI 설정
+  // apiKey는 Sonamu.secrets에서 로드되므로 여기서는 빈 문자열
+  openai: {
+    apiKey: "",
+    baseUrl: "https://api.openai.com/v1/embeddings",
+    model: "text-embedding-3-small",
+    dimensions: 1536,
+    maxTokens: 8191,
+    batchSize: 100,
+  },
+
+  // 청킹 설정 (필요시 사용)
+  chunking: {
+    chunkSize: 500,
+    chunkOverlap: 50,
+    minChunkSize: 50,
+    skipThreshold: 200,
+    separators: ["\n\n", "\n", "。", ". ", "! ", "? ", ", ", " "],
+  },
+
+  // 검색 설정
+  search: {
+    defaultLimit: 10,
+    similarityThreshold: 0.5,
+    vectorWeight: 0.7,
+    ftsWeight: 0.3,
+  },
+
+  // pgvector 설정
+  pgvector: {
+    iterativeScan: true,
+    efSearch: 100,
+  },
+};
+
+/**
+ * 설정 생성 헬퍼 함수
+ * 부분 설정만 제공하면 나머지는 기본값 사용
+ */
+export function createVectorConfig(
+  overrides: Partial<VectorConfig> = {}
+): VectorConfig {
+  return {
+    voyage: { ...DEFAULT_VECTOR_CONFIG.voyage, ...overrides.voyage },
+    openai: { ...DEFAULT_VECTOR_CONFIG.openai, ...overrides.openai },
+    chunking: { ...DEFAULT_VECTOR_CONFIG.chunking, ...overrides.chunking },
+    search: { ...DEFAULT_VECTOR_CONFIG.search, ...overrides.search },
+    pgvector: { ...DEFAULT_VECTOR_CONFIG.pgvector, ...overrides.pgvector },
+  };
+}

package/src/vector/embedding.ts

@@ -0,0 +1,193 @@
+import { Sonamu } from "../api/sonamu";
+import { DEFAULT_VECTOR_CONFIG } from "./config";
+import type {
+  EmbeddingProvider,
+  EmbeddingResult,
+  ProgressCallback,
+  VectorConfig,
+  VectorInputType,
+} from "./types";
+
+/**
+ * 임베딩 클라이언트
+ * Voyage AI와 OpenAI 임베딩을 통합 지원
+ */
+export class Embedding {
+  private config: VectorConfig;
+
+  constructor(config: Partial<VectorConfig> = {}) {
+    this.config = {
+      voyage: { ...DEFAULT_VECTOR_CONFIG.voyage, ...config.voyage },
+      openai: { ...DEFAULT_VECTOR_CONFIG.openai, ...config.openai },
+      chunking: { ...DEFAULT_VECTOR_CONFIG.chunking, ...config.chunking },
+      search: { ...DEFAULT_VECTOR_CONFIG.search, ...config.search },
+      pgvector: { ...DEFAULT_VECTOR_CONFIG.pgvector, ...config.pgvector },
+    };
+  }
+
+  /**
+   * 텍스트 임베딩 생성
+   * @param texts - 임베딩할 텍스트 배열
+   * @param provider - 'voyage' | 'openai'
+   * @param inputType - 'document' | 'query' (Voyage AI만 해당)
+   */
+  async embed(
+    texts: string[],
+    provider: EmbeddingProvider,
+    inputType: VectorInputType = "document"
+  ): Promise<EmbeddingResult[]> {
+    if (provider === "voyage") {
+      return this.embedVoyage(texts, inputType);
+    } else {
+      return this.embedOpenAI(texts);
+    }
+  }
+
+  /**
+   * 단일 텍스트 임베딩 (편의 메서드)
+   */
+  async embedOne(
+    text: string,
+    provider: EmbeddingProvider,
+    inputType: VectorInputType = "document"
+  ): Promise<EmbeddingResult> {
+    const results = await this.embed([text], provider, inputType);
+    return results[0];
+  }
+
+  /**
+   * Voyage AI 임베딩
+   */
+  private async embedVoyage(
+    texts: string[],
+    inputType: VectorInputType
+  ): Promise<EmbeddingResult[]> {
+    const voyageConfig = this.config.voyage;
+
+    // config에서 설정된 apiKey 우선, 없으면 Sonamu.secrets에서 로드
+    const apiKey = voyageConfig.apiKey || Sonamu.secrets?.voyage_api_key;
+    if (!apiKey) {
+      throw new Error(
+        "VOYAGE_API_KEY가 설정되지 않았습니다. 환경변수를 확인하세요."
+      );
+    }
+
+    const response = await fetch(voyageConfig.baseUrl, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${apiKey}`,
+      },
+      body: JSON.stringify({
+        input: texts,
+        model: voyageConfig.model,
+        input_type: inputType,
+      }),
+    });
+
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(`Voyage API error: ${response.status} - ${error}`);
+    }
+
+    const data = await response.json();
+
+    return data.data.map((item: { embedding: number[] }) => ({
+      embedding: item.embedding,
+      model: voyageConfig.model,
+      tokenCount: data.usage?.total_tokens || 0,
+    }));
+  }
+
+  /**
+   * OpenAI 임베딩
+   */
+  private async embedOpenAI(texts: string[]): Promise<EmbeddingResult[]> {
+    const openaiConfig = this.config.openai;
+
+    // config에서 설정된 apiKey 우선, 없으면 Sonamu.secrets에서 로드
+    const apiKey = openaiConfig.apiKey || Sonamu.secrets?.openai_api_key;
+    if (!apiKey) {
+      throw new Error(
+        "OPENAI_API_KEY가 설정되지 않았습니다. 환경변수를 확인하세요."
+      );
+    }
+
+    const response = await fetch(openaiConfig.baseUrl, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${apiKey}`,
+      },
+      body: JSON.stringify({
+        input: texts,
+        model: openaiConfig.model,
+      }),
+    });
+
+    if (!response.ok) {
+      const error = await response.text();
+      throw new Error(`OpenAI API error: ${response.status} - ${error}`);
+    }
+
+    const data = await response.json();
+
+    return data.data.map((item: { embedding: number[] }) => ({
+      embedding: item.embedding,
+      model: openaiConfig.model,
+      tokenCount: data.usage?.total_tokens || 0,
+    }));
+  }
+
+  /**
+   * 배치 임베딩 (대량 처리)
+   */
+  async embedBatch(
+    texts: string[],
+    provider: EmbeddingProvider,
+    inputType: VectorInputType = "document",
+    onProgress?: ProgressCallback
+  ): Promise<EmbeddingResult[]> {
+    const batchSize =
+      provider === "voyage"
+        ? this.config.voyage.batchSize
+        : this.config.openai.batchSize;
+
+    const results: EmbeddingResult[] = [];
+
+    for (let i = 0; i < texts.length; i += batchSize) {
+      const batch = texts.slice(i, i + batchSize);
+      const batchResults = await this.embed(batch, provider, inputType);
+      results.push(...batchResults);
+
+      onProgress?.(Math.min(i + batchSize, texts.length), texts.length);
+
+      // Rate limiting (100ms between batches)
+      if (i + batchSize < texts.length) {
+        await this.delay(100);
+      }
+    }
+
+    return results;
+  }
+
+  /**
+   * 벡터를 PostgreSQL vector 타입 문자열로 변환
+   */
+  static toVectorString(embedding: number[]): string {
+    return `[${embedding.join(",")}]`;
+  }
+
+  /**
+   * 임베딩 provider의 차원 수 반환
+   */
+  getDimensions(provider: EmbeddingProvider): number {
+    return provider === "voyage"
+      ? this.config.voyage.dimensions
+      : this.config.openai.dimensions;
+  }
+
+  private delay(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+}

package/src/vector/types.ts

@@ -0,0 +1,122 @@
+/**
+ * pgvector 통합을 위한 타입 정의
+ */
+
+/** 임베딩 제공자 */
+export type EmbeddingProvider = "voyage" | "openai";
+
+/** 입력 타입 (Voyage AI 전용 - 비대칭 임베딩) */
+export type VectorInputType = "document" | "query";
+
+/** 임베딩 결과 */
+export interface EmbeddingResult {
+  embedding: number[];
+  model: string;
+  tokenCount: number;
+}
+
+/** 청크 정보 */
+export interface Chunk {
+  index: number;
+  text: string;
+  startOffset: number;
+  endOffset: number;
+}
+
+/** 벡터 검색 결과 */
+export interface VectorSearchResult<T = Record<string, unknown>> {
+  id: number | string;
+  similarity: number;
+  data: T;
+}
+
+/** 하이브리드 검색 결과 (Vector + FTS) */
+export interface HybridSearchResult<T = Record<string, unknown>>
+  extends VectorSearchResult<T> {
+  vectorScore?: number;
+  ftsScore?: number;
+}
+
+/** 벤치마크 결과 */
+export interface BenchmarkResult {
+  provider: EmbeddingProvider;
+  embedTime: number;
+  searchTime: number;
+  results: VectorSearchResult[];
+}
+
+/** Voyage AI 설정 */
+export interface VoyageConfig {
+  apiKey: string;
+  baseUrl: string;
+  model: string;
+  dimensions: number;
+  maxTokens: number;
+  batchSize: number;
+}
+
+/** OpenAI 설정 */
+export interface OpenAIConfig {
+  apiKey: string;
+  baseUrl: string;
+  model: string;
+  dimensions: number;
+  maxTokens: number;
+  batchSize: number;
+}
+
+/** 청킹 설정 */
+export interface ChunkingConfig {
+  chunkSize: number;
+  chunkOverlap: number;
+  minChunkSize: number;
+  skipThreshold: number;
+  separators: string[];
+}
+
+/** 검색 설정 */
+export interface SearchConfig {
+  defaultLimit: number;
+  similarityThreshold: number;
+  vectorWeight: number;
+  ftsWeight: number;
+}
+
+/** pgvector 설정 */
+export interface PgvectorConfig {
+  iterativeScan: boolean;
+  efSearch: number;
+}
+
+/** 전체 벡터 설정 */
+export interface VectorConfig {
+  voyage: VoyageConfig;
+  openai: OpenAIConfig;
+  chunking: ChunkingConfig;
+  search: SearchConfig;
+  pgvector: PgvectorConfig;
+}
+
+/** 벡터 검색 옵션 */
+export interface VectorSearchOptions {
+  embeddingColumn?: string;
+  limit?: number;
+  threshold?: number;
+  where?: string;
+}
+
+/** 하이브리드 검색 옵션 */
+export interface HybridSearchOptions extends VectorSearchOptions {
+  vectorWeight?: number;
+  ftsWeight?: number;
+  ftsColumn?: string;
+}
+
+/** 임베딩 저장 항목 */
+export interface EmbeddingItem {
+  id: number;
+  text: string;
+}
+
+/** 진행률 콜백 */
+export type ProgressCallback = (processed: number, total: number) => void;