sf-vector-sdk 0.2.0__py3-none-any.whl

vector_sdk/content_types.py

@@ -0,0 +1,233 @@
+"""
+Content Type Registry
+
+Defines the pre-configured embedding settings for each content type.
+This ensures consistent model selection, dimensions, and storage across all SDK consumers.
+"""
+
+from dataclasses import dataclass
+from enum import IntEnum
+
+
+class ContentType(IntEnum):
+    """Content types that can be embedded"""
+    UNSPECIFIED = 0
+    TOPIC = 1
+    FLASHCARD = 2
+    TEST_QUESTION = 3
+    SPACED_TEST_QUESTION = 4
+    AUDIO_RECAP = 5
+    DOCUMENT = 6
+    NOTE = 7
+
+
+class EmbeddingModel(IntEnum):
+    """Vertex AI embedding models"""
+    UNSPECIFIED = 0
+    GEMINI_001 = 1      # gemini-embedding-001 - 3072 dimensions
+    TEXT_004 = 2        # text-embedding-004 - 768 dimensions
+    MULTILINGUAL_002 = 3  # text-multilingual-embedding-002
+
+
+class Priority(IntEnum):
+    """Queue priority levels"""
+    UNSPECIFIED = 0
+    CRITICAL = 1  # Reserved quota, immediate processing
+    HIGH = 2      # Processed before normal
+    NORMAL = 3    # Standard processing
+    LOW = 4       # Processed when capacity available
+
+
+@dataclass
+class MongoDBStorageConfig:
+    """MongoDB storage configuration"""
+    database: str
+    collection: str
+    embedding_field: str
+    upsert_key: str
+
+
+@dataclass
+class TurboPufferStorageConfig:
+    """TurboPuffer storage configuration"""
+    namespace: str
+    id_field: str
+    metadata_fields: list[str]
+
+
+@dataclass
+class ContentTypeConfig:
+    """Configuration for a content type's embedding settings"""
+    content_type: ContentType
+    model: EmbeddingModel
+    dimensions: int
+    default_priority: Priority
+    mongodb: MongoDBStorageConfig
+    turbopuffer: TurboPufferStorageConfig
+
+
+# Pre-configured settings for all supported content types
+CONTENT_TYPE_CONFIGS: dict[ContentType, ContentTypeConfig] = {
+    ContentType.TOPIC: ContentTypeConfig(
+        content_type=ContentType.TOPIC,
+        model=EmbeddingModel.GEMINI_001,
+        dimensions=3072,
+        default_priority=Priority.NORMAL,
+        mongodb=MongoDBStorageConfig(
+            database="events_new",
+            collection="topic_vectors",
+            embedding_field="topicDescriptionEmbedding",
+            upsert_key="combinationId",
+        ),
+        turbopuffer=TurboPufferStorageConfig(
+            namespace="topic_vectors",
+            id_field="_id",
+            metadata_fields=["topicId", "userId", "contentHash"],
+        ),
+    ),
+    ContentType.FLASHCARD: ContentTypeConfig(
+        content_type=ContentType.FLASHCARD,
+        model=EmbeddingModel.GEMINI_001,
+        dimensions=3072,
+        default_priority=Priority.HIGH,
+        mongodb=MongoDBStorageConfig(
+            database="events_new",
+            collection="tool_vectors",
+            embedding_field="toolEmbedding",
+            upsert_key="contentHash",
+        ),
+        turbopuffer=TurboPufferStorageConfig(
+            namespace="tool_vectors",
+            id_field="_id",
+            metadata_fields=["toolId", "toolCollection", "topicId", "userId"],
+        ),
+    ),
+    ContentType.TEST_QUESTION: ContentTypeConfig(
+        content_type=ContentType.TEST_QUESTION,
+        model=EmbeddingModel.GEMINI_001,
+        dimensions=3072,
+        default_priority=Priority.HIGH,
+        mongodb=MongoDBStorageConfig(
+            database="events_new",
+            collection="tool_vectors",
+            embedding_field="toolEmbedding",
+            upsert_key="contentHash",
+        ),
+        turbopuffer=TurboPufferStorageConfig(
+            namespace="tool_vectors",
+            id_field="_id",
+            metadata_fields=["toolId", "toolCollection", "topicId", "userId"],
+        ),
+    ),
+    ContentType.SPACED_TEST_QUESTION: ContentTypeConfig(
+        content_type=ContentType.SPACED_TEST_QUESTION,
+        model=EmbeddingModel.GEMINI_001,
+        dimensions=3072,
+        default_priority=Priority.NORMAL,
+        mongodb=MongoDBStorageConfig(
+            database="events_new",
+            collection="tool_vectors",
+            embedding_field="toolEmbedding",
+            upsert_key="contentHash",
+        ),
+        turbopuffer=TurboPufferStorageConfig(
+            namespace="tool_vectors",
+            id_field="_id",
+            metadata_fields=["toolId", "toolCollection", "topicId", "userId"],
+        ),
+    ),
+    ContentType.AUDIO_RECAP: ContentTypeConfig(
+        content_type=ContentType.AUDIO_RECAP,
+        model=EmbeddingModel.GEMINI_001,
+        dimensions=3072,
+        default_priority=Priority.NORMAL,
+        mongodb=MongoDBStorageConfig(
+            database="events_new",
+            collection="tool_vectors",
+            embedding_field="toolEmbedding",
+            upsert_key="contentHash",
+        ),
+        turbopuffer=TurboPufferStorageConfig(
+            namespace="tool_vectors",
+            id_field="_id",
+            metadata_fields=["toolId", "toolCollection", "topicId", "userId"],
+        ),
+    ),
+    ContentType.DOCUMENT: ContentTypeConfig(
+        content_type=ContentType.DOCUMENT,
+        model=EmbeddingModel.GEMINI_001,
+        dimensions=3072,
+        default_priority=Priority.NORMAL,
+        mongodb=MongoDBStorageConfig(
+            database="events_new",
+            collection="document_vectors",
+            embedding_field="documentEmbedding",
+            upsert_key="documentId",
+        ),
+        turbopuffer=TurboPufferStorageConfig(
+            namespace="document_vectors",
+            id_field="_id",
+            metadata_fields=["documentId", "userId", "sourceType"],
+        ),
+    ),
+    ContentType.NOTE: ContentTypeConfig(
+        content_type=ContentType.NOTE,
+        model=EmbeddingModel.GEMINI_001,
+        dimensions=3072,
+        default_priority=Priority.LOW,
+        mongodb=MongoDBStorageConfig(
+            database="events_new",
+            collection="note_vectors",
+            embedding_field="noteEmbedding",
+            upsert_key="noteId",
+        ),
+        turbopuffer=TurboPufferStorageConfig(
+            namespace="note_vectors",
+            id_field="_id",
+            metadata_fields=["noteId", "userId", "topicId"],
+        ),
+    ),
+}
+
+
+def get_content_type_config(content_type: ContentType) -> ContentTypeConfig:
+    """
+    Get the configuration for a content type.
+
+    Args:
+        content_type: The content type to get config for
+
+    Returns:
+        The configuration for the content type
+
+    Raises:
+        ValueError: If the content type is not supported
+    """
+    if content_type not in CONTENT_TYPE_CONFIGS:
+        raise ValueError(f"Unsupported content type: {content_type}")
+    return CONTENT_TYPE_CONFIGS[content_type]
+
+
+def get_model_name(model: EmbeddingModel) -> str:
+    """Get the Vertex AI model name string"""
+    model_names = {
+        EmbeddingModel.GEMINI_001: "gemini-embedding-001",
+        EmbeddingModel.TEXT_004: "text-embedding-004",
+        EmbeddingModel.MULTILINGUAL_002: "text-multilingual-embedding-002",
+    }
+    return model_names.get(model, "gemini-embedding-001")
+
+
+def get_priority_string(priority: Priority) -> str:
+    """Get the priority string for Redis streams"""
+    priority_strings = {
+        Priority.CRITICAL: "critical",
+        Priority.HIGH: "high",
+        Priority.NORMAL: "normal",
+        Priority.LOW: "low",
+    }
+    return priority_strings.get(priority, "normal")
+
+
+# All supported content types (excluding UNSPECIFIED)
+SUPPORTED_CONTENT_TYPES = [ct for ct in ContentType if ct != ContentType.UNSPECIFIED]

vector_sdk/generated/embedding_pipeline/content_types/v1/content_types_pb2.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# NO CHECKED-IN PROTOBUF GENCODE
+# source: embedding_pipeline/content_types/v1/content_types.proto
+# Protobuf Python Version: 6.33.4
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import runtime_version as _runtime_version
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+_runtime_version.ValidateProtobufRuntimeVersion(
+    _runtime_version.Domain.PUBLIC,
+    6,
+    33,
+    4,
+    '',
+    'embedding_pipeline/content_types/v1/content_types.proto'
+)
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n7embedding_pipeline/content_types/v1/content_types.proto\x12#embedding_pipeline.content_types.v1\"\xbb\x04\n\x11\x43ontentTypeConfig\x12S\n\x0c\x63ontent_type\x18\x01 \x01(\x0e\x32\x30.embedding_pipeline.content_types.v1.ContentTypeR\x0b\x63ontentType\x12I\n\x05model\x18\x02 \x01(\x0e\x32\x33.embedding_pipeline.content_types.v1.EmbeddingModelR\x05model\x12\x1e\n\ndimensions\x18\x03 \x01(\x05R\ndimensions\x12X\n\x10\x64\x65\x66\x61ult_priority\x18\x04 \x01(\x0e\x32-.embedding_pipeline.content_types.v1.PriorityR\x0f\x64\x65\x66\x61ultPriority\x12S\n\x07mongodb\x18\x05 \x01(\x0b\x32\x39.embedding_pipeline.content_types.v1.MongoDBStorageConfigR\x07mongodb\x12_\n\x0bturbopuffer\x18\x06 \x01(\x0b\x32=.embedding_pipeline.content_types.v1.TurboPufferStorageConfigR\x0bturbopuffer\x12V\n\x08pinecone\x18\x07 \x01(\x0b\x32:.embedding_pipeline.content_types.v1.PineconeStorageConfigR\x08pinecone\"\x9a\x01\n\x14MongoDBStorageConfig\x12\x1a\n\x08\x64\x61tabase\x18\x01 \x01(\tR\x08\x64\x61tabase\x12\x1e\n\ncollection\x18\x02 \x01(\tR\ncollection\x12\'\n\x0f\x65mbedding_field\x18\x03 \x01(\tR\x0e\x65mbeddingField\x12\x1d\n\nupsert_key\x18\x04 \x01(\tR\tupsertKey\"|\n\x18TurboPufferStorageConfig\x12\x1c\n\tnamespace\x18\x01 \x01(\tR\tnamespace\x12\x19\n\x08id_field\x18\x02 \x01(\tR\x07idField\x12\'\n\x0fmetadata_fields\x18\x03 \x03(\tR\x0emetadataFields\"\x98\x01\n\x15PineconeStorageConfig\x12\x1d\n\nindex_name\x18\x01 \x01(\tR\tindexName\x12\x1c\n\tnamespace\x18\x02 \x01(\tR\tnamespace\x12\x19\n\x08id_field\x18\x03 \x01(\tR\x07idField\x12\'\n\x0fmetadata_fields\x18\x04 \x03(\tR\x0emetadataFields\"|\n\x0f\x45mbeddingConfig\x12I\n\x05model\x18\x01 \x01(\x0e\x32\x33.embedding_pipeline.content_types.v1.EmbeddingModelR\x05model\x12\x1e\n\ndimensions\x18\x02 \x01(\x05R\ndimensions\"g\n\x13\x43ontentTypeRegistry\x12P\n\x07\x63onfigs\x18\x01 \x03(\x0b\x32\x36.embedding_pipeline.content_types.v1.ContentTypeConfigR\x07\x63onfigs*\xf6\x01\n\x0b\x43ontentType\x12\x1c\n\x18\x43ONTENT_TYPE_UNSPECIFIED\x10\x00\x12\x16\n\x12\x43ONTENT_TYPE_TOPIC\x10\x01\x12\x1a\n\x16\x43ONTENT_TYPE_FLASHCARD\x10\x02\x12\x1e\n\x1a\x43ONTENT_TYPE_TEST_QUESTION\x10\x03\x12%\n!CONTENT_TYPE_SPACED_TEST_QUESTION\x10\x04\x12\x1c\n\x18\x43ONTENT_TYPE_AUDIO_RECAP\x10\x05\x12\x19\n\x15\x43ONTENT_TYPE_DOCUMENT\x10\x06\x12\x15\n\x11\x43ONTENT_TYPE_NOTE\x10\x07*u\n\x11\x45mbeddingProvider\x12\"\n\x1e\x45MBEDDING_PROVIDER_UNSPECIFIED\x10\x00\x12\x1d\n\x19\x45MBEDDING_PROVIDER_GOOGLE\x10\x01\x12\x1d\n\x19\x45MBEDDING_PROVIDER_OPENAI\x10\x02*\xdd\x01\n\x0e\x45mbeddingModel\x12\x1f\n\x1b\x45MBEDDING_MODEL_UNSPECIFIED\x10\x00\x12\x1e\n\x1a\x45MBEDDING_MODEL_GEMINI_001\x10\x01\x12\x1c\n\x18\x45MBEDDING_MODEL_TEXT_004\x10\x02\x12$\n EMBEDDING_MODEL_MULTILINGUAL_002\x10\x03\x12\"\n\x1e\x45MBEDDING_MODEL_OPENAI_3_SMALL\x10\n\x12\"\n\x1e\x45MBEDDING_MODEL_OPENAI_3_LARGE\x10\x0b*u\n\x08Priority\x12\x18\n\x14PRIORITY_UNSPECIFIED\x10\x00\x12\x15\n\x11PRIORITY_CRITICAL\x10\x01\x12\x11\n\rPRIORITY_HIGH\x10\x02\x12\x13\n\x0fPRIORITY_NORMAL\x10\x03\x12\x10\n\x0cPRIORITY_LOW\x10\x04*\x8d\x01\n\x0eVectorDatabase\x12\x1f\n\x1bVECTOR_DATABASE_UNSPECIFIED\x10\x00\x12\x1b\n\x17VECTOR_DATABASE_MONGODB\x10\x01\x12\x1f\n\x1bVECTOR_DATABASE_TURBOPUFFER\x10\x02\x12\x1c\n\x18VECTOR_DATABASE_PINECONE\x10\x03\x42oZmgithub.com/GoStudyFetchGo/vector-management-monorepo/packages/go/proto-go/embedding_pipeline/content_types/v1b\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'embedding_pipeline.content_types.v1.content_types_pb2', _globals)
+if not _descriptor._USE_C_DESCRIPTORS:
+  _globals['DESCRIPTOR']._loaded_options = None
+  _globals['DESCRIPTOR']._serialized_options = b'Zmgithub.com/GoStudyFetchGo/vector-management-monorepo/packages/go/proto-go/embedding_pipeline/content_types/v1'
+  _globals['_CONTENTTYPE']._serialized_start=1340
+  _globals['_CONTENTTYPE']._serialized_end=1586
+  _globals['_EMBEDDINGPROVIDER']._serialized_start=1588
+  _globals['_EMBEDDINGPROVIDER']._serialized_end=1705
+  _globals['_EMBEDDINGMODEL']._serialized_start=1708
+  _globals['_EMBEDDINGMODEL']._serialized_end=1929
+  _globals['_PRIORITY']._serialized_start=1931
+  _globals['_PRIORITY']._serialized_end=2048
+  _globals['_VECTORDATABASE']._serialized_start=2051
+  _globals['_VECTORDATABASE']._serialized_end=2192
+  _globals['_CONTENTTYPECONFIG']._serialized_start=97
+  _globals['_CONTENTTYPECONFIG']._serialized_end=668
+  _globals['_MONGODBSTORAGECONFIG']._serialized_start=671
+  _globals['_MONGODBSTORAGECONFIG']._serialized_end=825
+  _globals['_TURBOPUFFERSTORAGECONFIG']._serialized_start=827
+  _globals['_TURBOPUFFERSTORAGECONFIG']._serialized_end=951
+  _globals['_PINECONESTORAGECONFIG']._serialized_start=954
+  _globals['_PINECONESTORAGECONFIG']._serialized_end=1106
+  _globals['_EMBEDDINGCONFIG']._serialized_start=1108
+  _globals['_EMBEDDINGCONFIG']._serialized_end=1232
+  _globals['_CONTENTTYPEREGISTRY']._serialized_start=1234
+  _globals['_CONTENTTYPEREGISTRY']._serialized_end=1337
+# @@protoc_insertion_point(module_scope)

vector_sdk/generated/embedding_pipeline/content_types/v1/content_types_pb2.pyi

@@ -0,0 +1,141 @@
+from google.protobuf.internal import containers as _containers
+from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from collections.abc import Iterable as _Iterable, Mapping as _Mapping
+from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union
+
+DESCRIPTOR: _descriptor.FileDescriptor
+
+class ContentType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = ()
+    CONTENT_TYPE_UNSPECIFIED: _ClassVar[ContentType]
+    CONTENT_TYPE_TOPIC: _ClassVar[ContentType]
+    CONTENT_TYPE_FLASHCARD: _ClassVar[ContentType]
+    CONTENT_TYPE_TEST_QUESTION: _ClassVar[ContentType]
+    CONTENT_TYPE_SPACED_TEST_QUESTION: _ClassVar[ContentType]
+    CONTENT_TYPE_AUDIO_RECAP: _ClassVar[ContentType]
+    CONTENT_TYPE_DOCUMENT: _ClassVar[ContentType]
+    CONTENT_TYPE_NOTE: _ClassVar[ContentType]
+
+class EmbeddingProvider(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = ()
+    EMBEDDING_PROVIDER_UNSPECIFIED: _ClassVar[EmbeddingProvider]
+    EMBEDDING_PROVIDER_GOOGLE: _ClassVar[EmbeddingProvider]
+    EMBEDDING_PROVIDER_OPENAI: _ClassVar[EmbeddingProvider]
+
+class EmbeddingModel(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = ()
+    EMBEDDING_MODEL_UNSPECIFIED: _ClassVar[EmbeddingModel]
+    EMBEDDING_MODEL_GEMINI_001: _ClassVar[EmbeddingModel]
+    EMBEDDING_MODEL_TEXT_004: _ClassVar[EmbeddingModel]
+    EMBEDDING_MODEL_MULTILINGUAL_002: _ClassVar[EmbeddingModel]
+    EMBEDDING_MODEL_OPENAI_3_SMALL: _ClassVar[EmbeddingModel]
+    EMBEDDING_MODEL_OPENAI_3_LARGE: _ClassVar[EmbeddingModel]
+
+class Priority(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = ()
+    PRIORITY_UNSPECIFIED: _ClassVar[Priority]
+    PRIORITY_CRITICAL: _ClassVar[Priority]
+    PRIORITY_HIGH: _ClassVar[Priority]
+    PRIORITY_NORMAL: _ClassVar[Priority]
+    PRIORITY_LOW: _ClassVar[Priority]
+
+class VectorDatabase(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = ()
+    VECTOR_DATABASE_UNSPECIFIED: _ClassVar[VectorDatabase]
+    VECTOR_DATABASE_MONGODB: _ClassVar[VectorDatabase]
+    VECTOR_DATABASE_TURBOPUFFER: _ClassVar[VectorDatabase]
+    VECTOR_DATABASE_PINECONE: _ClassVar[VectorDatabase]
+CONTENT_TYPE_UNSPECIFIED: ContentType
+CONTENT_TYPE_TOPIC: ContentType
+CONTENT_TYPE_FLASHCARD: ContentType
+CONTENT_TYPE_TEST_QUESTION: ContentType
+CONTENT_TYPE_SPACED_TEST_QUESTION: ContentType
+CONTENT_TYPE_AUDIO_RECAP: ContentType
+CONTENT_TYPE_DOCUMENT: ContentType
+CONTENT_TYPE_NOTE: ContentType
+EMBEDDING_PROVIDER_UNSPECIFIED: EmbeddingProvider
+EMBEDDING_PROVIDER_GOOGLE: EmbeddingProvider
+EMBEDDING_PROVIDER_OPENAI: EmbeddingProvider
+EMBEDDING_MODEL_UNSPECIFIED: EmbeddingModel
+EMBEDDING_MODEL_GEMINI_001: EmbeddingModel
+EMBEDDING_MODEL_TEXT_004: EmbeddingModel
+EMBEDDING_MODEL_MULTILINGUAL_002: EmbeddingModel
+EMBEDDING_MODEL_OPENAI_3_SMALL: EmbeddingModel
+EMBEDDING_MODEL_OPENAI_3_LARGE: EmbeddingModel
+PRIORITY_UNSPECIFIED: Priority
+PRIORITY_CRITICAL: Priority
+PRIORITY_HIGH: Priority
+PRIORITY_NORMAL: Priority
+PRIORITY_LOW: Priority
+VECTOR_DATABASE_UNSPECIFIED: VectorDatabase
+VECTOR_DATABASE_MONGODB: VectorDatabase
+VECTOR_DATABASE_TURBOPUFFER: VectorDatabase
+VECTOR_DATABASE_PINECONE: VectorDatabase
+
+class ContentTypeConfig(_message.Message):
+    __slots__ = ("content_type", "model", "dimensions", "default_priority", "mongodb", "turbopuffer", "pinecone")
+    CONTENT_TYPE_FIELD_NUMBER: _ClassVar[int]
+    MODEL_FIELD_NUMBER: _ClassVar[int]
+    DIMENSIONS_FIELD_NUMBER: _ClassVar[int]
+    DEFAULT_PRIORITY_FIELD_NUMBER: _ClassVar[int]
+    MONGODB_FIELD_NUMBER: _ClassVar[int]
+    TURBOPUFFER_FIELD_NUMBER: _ClassVar[int]
+    PINECONE_FIELD_NUMBER: _ClassVar[int]
+    content_type: ContentType
+    model: EmbeddingModel
+    dimensions: int
+    default_priority: Priority
+    mongodb: MongoDBStorageConfig
+    turbopuffer: TurboPufferStorageConfig
+    pinecone: PineconeStorageConfig
+    def __init__(self, content_type: _Optional[_Union[ContentType, str]] = ..., model: _Optional[_Union[EmbeddingModel, str]] = ..., dimensions: _Optional[int] = ..., default_priority: _Optional[_Union[Priority, str]] = ..., mongodb: _Optional[_Union[MongoDBStorageConfig, _Mapping]] = ..., turbopuffer: _Optional[_Union[TurboPufferStorageConfig, _Mapping]] = ..., pinecone: _Optional[_Union[PineconeStorageConfig, _Mapping]] = ...) -> None: ...
+
+class MongoDBStorageConfig(_message.Message):
+    __slots__ = ("database", "collection", "embedding_field", "upsert_key")
+    DATABASE_FIELD_NUMBER: _ClassVar[int]
+    COLLECTION_FIELD_NUMBER: _ClassVar[int]
+    EMBEDDING_FIELD_FIELD_NUMBER: _ClassVar[int]
+    UPSERT_KEY_FIELD_NUMBER: _ClassVar[int]
+    database: str
+    collection: str
+    embedding_field: str
+    upsert_key: str
+    def __init__(self, database: _Optional[str] = ..., collection: _Optional[str] = ..., embedding_field: _Optional[str] = ..., upsert_key: _Optional[str] = ...) -> None: ...
+
+class TurboPufferStorageConfig(_message.Message):
+    __slots__ = ("namespace", "id_field", "metadata_fields")
+    NAMESPACE_FIELD_NUMBER: _ClassVar[int]
+    ID_FIELD_FIELD_NUMBER: _ClassVar[int]
+    METADATA_FIELDS_FIELD_NUMBER: _ClassVar[int]
+    namespace: str
+    id_field: str
+    metadata_fields: _containers.RepeatedScalarFieldContainer[str]
+    def __init__(self, namespace: _Optional[str] = ..., id_field: _Optional[str] = ..., metadata_fields: _Optional[_Iterable[str]] = ...) -> None: ...
+
+class PineconeStorageConfig(_message.Message):
+    __slots__ = ("index_name", "namespace", "id_field", "metadata_fields")
+    INDEX_NAME_FIELD_NUMBER: _ClassVar[int]
+    NAMESPACE_FIELD_NUMBER: _ClassVar[int]
+    ID_FIELD_FIELD_NUMBER: _ClassVar[int]
+    METADATA_FIELDS_FIELD_NUMBER: _ClassVar[int]
+    index_name: str
+    namespace: str
+    id_field: str
+    metadata_fields: _containers.RepeatedScalarFieldContainer[str]
+    def __init__(self, index_name: _Optional[str] = ..., namespace: _Optional[str] = ..., id_field: _Optional[str] = ..., metadata_fields: _Optional[_Iterable[str]] = ...) -> None: ...
+
+class EmbeddingConfig(_message.Message):
+    __slots__ = ("model", "dimensions")
+    MODEL_FIELD_NUMBER: _ClassVar[int]
+    DIMENSIONS_FIELD_NUMBER: _ClassVar[int]
+    model: EmbeddingModel
+    dimensions: int
+    def __init__(self, model: _Optional[_Union[EmbeddingModel, str]] = ..., dimensions: _Optional[int] = ...) -> None: ...
+
+class ContentTypeRegistry(_message.Message):
+    __slots__ = ("configs",)
+    CONFIGS_FIELD_NUMBER: _ClassVar[int]
+    configs: _containers.RepeatedCompositeFieldContainer[ContentTypeConfig]
+    def __init__(self, configs: _Optional[_Iterable[_Union[ContentTypeConfig, _Mapping]]] = ...) -> None: ...

vector_sdk/generated/embedding_pipeline/db/vectors/v1/vectors_pb2.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# NO CHECKED-IN PROTOBUF GENCODE
+# source: embedding_pipeline/db/vectors/v1/vectors.proto
+# Protobuf Python Version: 6.33.4
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import runtime_version as _runtime_version
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+_runtime_version.ValidateProtobufRuntimeVersion(
+    _runtime_version.Domain.PUBLIC,
+    6,
+    33,
+    4,
+    '',
+    'embedding_pipeline/db/vectors/v1/vectors.proto'
+)
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.embedding_pipeline/db/vectors/v1/vectors.proto\x12 embedding_pipeline.db.vectors.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"\xfc\x02\n\x0e\x43lusterVersion\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x1c\n\talgorithm\x18\x02 \x01(\tR\talgorithm\x12\x18\n\x07version\x18\x03 \x01(\tR\x07version\x12`\n\nparameters\x18\x04 \x03(\x0b\x32@.embedding_pipeline.db.vectors.v1.ClusterVersion.ParametersEntryR\nparameters\x12%\n\x0etotal_clusters\x18\x05 \x01(\x05R\rtotalClusters\x12\x1f\n\x0btotal_items\x18\x06 \x01(\x05R\ntotalItems\x12\x39\n\ncreated_at\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tcreatedAt\x1a=\n\x0fParametersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\"\x8d\x04\n\x0bTopicVector\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x19\n\x08topic_id\x18\x02 \x01(\tR\x07topicId\x12\x17\n\x07user_id\x18\x03 \x01(\tR\x06userId\x12\x1c\n\tembedding\x18\x04 \x03(\x02R\tembedding\x12W\n\x08metadata\x18\x05 \x03(\x0b\x32;.embedding_pipeline.db.vectors.v1.TopicVector.MetadataEntryR\x08metadata\x12*\n\x0e\x63ombination_id\x18\x06 \x01(\tH\x00R\rcombinationId\x88\x01\x01\x12\x39\n\ncreated_at\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tcreatedAt\x12\x39\n\nupdated_at\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tupdatedAt\x12\x37\n\x15last_complete_version\x18\t \x01(\x05H\x01R\x13lastCompleteVersion\x88\x01\x01\x1a;\n\rMetadataEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x11\n\x0f_combination_idB\x18\n\x16_last_complete_version\"\xe9\x03\n\nToolVector\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x17\n\x07tool_id\x18\x02 \x01(\tR\x06toolId\x12\x19\n\x08topic_id\x18\x03 \x01(\tR\x07topicId\x12\'\n\x0ftool_collection\x18\x04 \x01(\tR\x0etoolCollection\x12\x1c\n\tembedding\x18\x05 \x03(\x02R\tembedding\x12V\n\x08metadata\x18\x06 \x03(\x0b\x32:.embedding_pipeline.db.vectors.v1.ToolVector.MetadataEntryR\x08metadata\x12#\n\x0btool_set_id\x18\x07 \x01(\tH\x00R\ttoolSetId\x88\x01\x01\x12\x33\n\x13tool_set_collection\x18\x08 \x01(\tH\x01R\x11toolSetCollection\x88\x01\x01\x12\x39\n\ncreated_at\x18\t \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tcreatedAt\x1a;\n\rMetadataEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x0e\n\x0c_tool_set_idB\x16\n\x14_tool_set_collection\"\x80\x05\n\x12TopicGroupEnhanced\x12\x0e\n\x02id\x18\x01 \x01(\tR\x02id\x12\x14\n\x05topic\x18\x02 \x01(\tR\x05topic\x12 \n\x0b\x64\x65scription\x18\x03 \x01(\tR\x0b\x64\x65scription\x12*\n\x0e\x63\x65ntroid_index\x18\x04 \x01(\x05H\x00R\rcentroidIndex\x88\x01\x01\x12&\n\x0csubject_type\x18\x05 \x01(\tH\x01R\x0bsubjectType\x88\x01\x01\x12+\n\x0ftopics_in_group\x18\x06 \x01(\x05H\x02R\rtopicsInGroup\x88\x01\x01\x12\x1d\n\x07version\x18\x07 \x01(\x05H\x03R\x07version\x88\x01\x01\x12\x46\n\x0enext_recluster\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x04R\rnextRecluster\x88\x01\x01\x12\"\n\ncluster_id\x18\t \x01(\tH\x05R\tclusterId\x88\x01\x01\x12(\n\rcluster_level\x18\n \x01(\x05H\x06R\x0c\x63lusterLevel\x88\x01\x01\x12\x39\n\ncreated_at\x18\x0b \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tcreatedAt\x12\x39\n\nupdated_at\x18\x0c \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tupdatedAtB\x11\n\x0f_centroid_indexB\x0f\n\r_subject_typeB\x12\n\x10_topics_in_groupB\n\n\x08_versionB\x11\n\x0f_next_reclusterB\r\n\x0b_cluster_idB\x10\n\x0e_cluster_level\"\xaf\x03\n\x0b\x43lusterNode\x12\x1d\n\ncluster_id\x18\x01 \x01(\tR\tclusterId\x12\x14\n\x05level\x18\x02 \x01(\x05R\x05level\x12 \n\tparent_id\x18\x03 \x01(\tH\x00R\x08parentId\x88\x01\x01\x12\x1a\n\x08\x63hildren\x18\x04 \x03(\tR\x08\x63hildren\x12\x1d\n\nitem_count\x18\x05 \x01(\x05R\titemCount\x12\x1d\n\x07summary\x18\x06 \x01(\tH\x01R\x07summary\x88\x01\x01\x12\x1a\n\x08\x63\x65ntroid\x18\x07 \x03(\x02R\x08\x63\x65ntroid\x12.\n\x10silhouette_score\x18\x08 \x01(\x01H\x02R\x0fsilhouetteScore\x88\x01\x01\x12\x39\n\ncreated_at\x18\t \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tcreatedAt\x12\x39\n\nupdated_at\x18\n \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tupdatedAtB\x0c\n\n_parent_idB\n\n\x08_summaryB\x13\n\x11_silhouette_scoreBlZjgithub.com/GoStudyFetchGo/vector-management-monorepo/packages/go/proto-go/embedding_pipeline/db/vectors/v1b\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'embedding_pipeline.db.vectors.v1.vectors_pb2', _globals)
+if not _descriptor._USE_C_DESCRIPTORS:
+  _globals['DESCRIPTOR']._loaded_options = None
+  _globals['DESCRIPTOR']._serialized_options = b'Zjgithub.com/GoStudyFetchGo/vector-management-monorepo/packages/go/proto-go/embedding_pipeline/db/vectors/v1'
+  _globals['_CLUSTERVERSION_PARAMETERSENTRY']._loaded_options = None
+  _globals['_CLUSTERVERSION_PARAMETERSENTRY']._serialized_options = b'8\001'
+  _globals['_TOPICVECTOR_METADATAENTRY']._loaded_options = None
+  _globals['_TOPICVECTOR_METADATAENTRY']._serialized_options = b'8\001'
+  _globals['_TOOLVECTOR_METADATAENTRY']._loaded_options = None
+  _globals['_TOOLVECTOR_METADATAENTRY']._serialized_options = b'8\001'
+  _globals['_CLUSTERVERSION']._serialized_start=118
+  _globals['_CLUSTERVERSION']._serialized_end=498
+  _globals['_CLUSTERVERSION_PARAMETERSENTRY']._serialized_start=437
+  _globals['_CLUSTERVERSION_PARAMETERSENTRY']._serialized_end=498
+  _globals['_TOPICVECTOR']._serialized_start=501
+  _globals['_TOPICVECTOR']._serialized_end=1026
+  _globals['_TOPICVECTOR_METADATAENTRY']._serialized_start=922
+  _globals['_TOPICVECTOR_METADATAENTRY']._serialized_end=981
+  _globals['_TOOLVECTOR']._serialized_start=1029
+  _globals['_TOOLVECTOR']._serialized_end=1518
+  _globals['_TOOLVECTOR_METADATAENTRY']._serialized_start=922
+  _globals['_TOOLVECTOR_METADATAENTRY']._serialized_end=981
+  _globals['_TOPICGROUPENHANCED']._serialized_start=1521
+  _globals['_TOPICGROUPENHANCED']._serialized_end=2161
+  _globals['_CLUSTERNODE']._serialized_start=2164
+  _globals['_CLUSTERNODE']._serialized_end=2595
+# @@protoc_insertion_point(module_scope)

vector_sdk/generated/embedding_pipeline/db/vectors/v1/vectors_pb2.pyi

@@ -0,0 +1,145 @@
+import datetime
+
+from google.protobuf import timestamp_pb2 as _timestamp_pb2
+from google.protobuf.internal import containers as _containers
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from collections.abc import Iterable as _Iterable, Mapping as _Mapping
+from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union
+
+DESCRIPTOR: _descriptor.FileDescriptor
+
+class ClusterVersion(_message.Message):
+    __slots__ = ("id", "algorithm", "version", "parameters", "total_clusters", "total_items", "created_at")
+    class ParametersEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: str
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
+    ID_FIELD_NUMBER: _ClassVar[int]
+    ALGORITHM_FIELD_NUMBER: _ClassVar[int]
+    VERSION_FIELD_NUMBER: _ClassVar[int]
+    PARAMETERS_FIELD_NUMBER: _ClassVar[int]
+    TOTAL_CLUSTERS_FIELD_NUMBER: _ClassVar[int]
+    TOTAL_ITEMS_FIELD_NUMBER: _ClassVar[int]
+    CREATED_AT_FIELD_NUMBER: _ClassVar[int]
+    id: str
+    algorithm: str
+    version: str
+    parameters: _containers.ScalarMap[str, str]
+    total_clusters: int
+    total_items: int
+    created_at: _timestamp_pb2.Timestamp
+    def __init__(self, id: _Optional[str] = ..., algorithm: _Optional[str] = ..., version: _Optional[str] = ..., parameters: _Optional[_Mapping[str, str]] = ..., total_clusters: _Optional[int] = ..., total_items: _Optional[int] = ..., created_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ...
+
+class TopicVector(_message.Message):
+    __slots__ = ("id", "topic_id", "user_id", "embedding", "metadata", "combination_id", "created_at", "updated_at", "last_complete_version")
+    class MetadataEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: str
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
+    ID_FIELD_NUMBER: _ClassVar[int]
+    TOPIC_ID_FIELD_NUMBER: _ClassVar[int]
+    USER_ID_FIELD_NUMBER: _ClassVar[int]
+    EMBEDDING_FIELD_NUMBER: _ClassVar[int]
+    METADATA_FIELD_NUMBER: _ClassVar[int]
+    COMBINATION_ID_FIELD_NUMBER: _ClassVar[int]
+    CREATED_AT_FIELD_NUMBER: _ClassVar[int]
+    UPDATED_AT_FIELD_NUMBER: _ClassVar[int]
+    LAST_COMPLETE_VERSION_FIELD_NUMBER: _ClassVar[int]
+    id: str
+    topic_id: str
+    user_id: str
+    embedding: _containers.RepeatedScalarFieldContainer[float]
+    metadata: _containers.ScalarMap[str, str]
+    combination_id: str
+    created_at: _timestamp_pb2.Timestamp
+    updated_at: _timestamp_pb2.Timestamp
+    last_complete_version: int
+    def __init__(self, id: _Optional[str] = ..., topic_id: _Optional[str] = ..., user_id: _Optional[str] = ..., embedding: _Optional[_Iterable[float]] = ..., metadata: _Optional[_Mapping[str, str]] = ..., combination_id: _Optional[str] = ..., created_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., updated_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., last_complete_version: _Optional[int] = ...) -> None: ...
+
+class ToolVector(_message.Message):
+    __slots__ = ("id", "tool_id", "topic_id", "tool_collection", "embedding", "metadata", "tool_set_id", "tool_set_collection", "created_at")
+    class MetadataEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: str
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
+    ID_FIELD_NUMBER: _ClassVar[int]
+    TOOL_ID_FIELD_NUMBER: _ClassVar[int]
+    TOPIC_ID_FIELD_NUMBER: _ClassVar[int]
+    TOOL_COLLECTION_FIELD_NUMBER: _ClassVar[int]
+    EMBEDDING_FIELD_NUMBER: _ClassVar[int]
+    METADATA_FIELD_NUMBER: _ClassVar[int]
+    TOOL_SET_ID_FIELD_NUMBER: _ClassVar[int]
+    TOOL_SET_COLLECTION_FIELD_NUMBER: _ClassVar[int]
+    CREATED_AT_FIELD_NUMBER: _ClassVar[int]
+    id: str
+    tool_id: str
+    topic_id: str
+    tool_collection: str
+    embedding: _containers.RepeatedScalarFieldContainer[float]
+    metadata: _containers.ScalarMap[str, str]
+    tool_set_id: str
+    tool_set_collection: str
+    created_at: _timestamp_pb2.Timestamp
+    def __init__(self, id: _Optional[str] = ..., tool_id: _Optional[str] = ..., topic_id: _Optional[str] = ..., tool_collection: _Optional[str] = ..., embedding: _Optional[_Iterable[float]] = ..., metadata: _Optional[_Mapping[str, str]] = ..., tool_set_id: _Optional[str] = ..., tool_set_collection: _Optional[str] = ..., created_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ...
+
+class TopicGroupEnhanced(_message.Message):
+    __slots__ = ("id", "topic", "description", "centroid_index", "subject_type", "topics_in_group", "version", "next_recluster", "cluster_id", "cluster_level", "created_at", "updated_at")
+    ID_FIELD_NUMBER: _ClassVar[int]
+    TOPIC_FIELD_NUMBER: _ClassVar[int]
+    DESCRIPTION_FIELD_NUMBER: _ClassVar[int]
+    CENTROID_INDEX_FIELD_NUMBER: _ClassVar[int]
+    SUBJECT_TYPE_FIELD_NUMBER: _ClassVar[int]
+    TOPICS_IN_GROUP_FIELD_NUMBER: _ClassVar[int]
+    VERSION_FIELD_NUMBER: _ClassVar[int]
+    NEXT_RECLUSTER_FIELD_NUMBER: _ClassVar[int]
+    CLUSTER_ID_FIELD_NUMBER: _ClassVar[int]
+    CLUSTER_LEVEL_FIELD_NUMBER: _ClassVar[int]
+    CREATED_AT_FIELD_NUMBER: _ClassVar[int]
+    UPDATED_AT_FIELD_NUMBER: _ClassVar[int]
+    id: str
+    topic: str
+    description: str
+    centroid_index: int
+    subject_type: str
+    topics_in_group: int
+    version: int
+    next_recluster: _timestamp_pb2.Timestamp
+    cluster_id: str
+    cluster_level: int
+    created_at: _timestamp_pb2.Timestamp
+    updated_at: _timestamp_pb2.Timestamp
+    def __init__(self, id: _Optional[str] = ..., topic: _Optional[str] = ..., description: _Optional[str] = ..., centroid_index: _Optional[int] = ..., subject_type: _Optional[str] = ..., topics_in_group: _Optional[int] = ..., version: _Optional[int] = ..., next_recluster: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., cluster_id: _Optional[str] = ..., cluster_level: _Optional[int] = ..., created_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., updated_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ...
+
+class ClusterNode(_message.Message):
+    __slots__ = ("cluster_id", "level", "parent_id", "children", "item_count", "summary", "centroid", "silhouette_score", "created_at", "updated_at")
+    CLUSTER_ID_FIELD_NUMBER: _ClassVar[int]
+    LEVEL_FIELD_NUMBER: _ClassVar[int]
+    PARENT_ID_FIELD_NUMBER: _ClassVar[int]
+    CHILDREN_FIELD_NUMBER: _ClassVar[int]
+    ITEM_COUNT_FIELD_NUMBER: _ClassVar[int]
+    SUMMARY_FIELD_NUMBER: _ClassVar[int]
+    CENTROID_FIELD_NUMBER: _ClassVar[int]
+    SILHOUETTE_SCORE_FIELD_NUMBER: _ClassVar[int]
+    CREATED_AT_FIELD_NUMBER: _ClassVar[int]
+    UPDATED_AT_FIELD_NUMBER: _ClassVar[int]
+    cluster_id: str
+    level: int
+    parent_id: str
+    children: _containers.RepeatedScalarFieldContainer[str]
+    item_count: int
+    summary: str
+    centroid: _containers.RepeatedScalarFieldContainer[float]
+    silhouette_score: float
+    created_at: _timestamp_pb2.Timestamp
+    updated_at: _timestamp_pb2.Timestamp
+    def __init__(self, cluster_id: _Optional[str] = ..., level: _Optional[int] = ..., parent_id: _Optional[str] = ..., children: _Optional[_Iterable[str]] = ..., item_count: _Optional[int] = ..., summary: _Optional[str] = ..., centroid: _Optional[_Iterable[float]] = ..., silhouette_score: _Optional[float] = ..., created_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ..., updated_at: _Optional[_Union[datetime.datetime, _timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ...