letta-nightly 0.8.9.dev20250705104147__py3-none-any.whl → 0.8.10.dev20250707035305__py3-none-any.whl

letta/schemas/message.py

@@ -84,11 +84,11 @@ class MessageCreate(BaseModel):
         description="The content of the message.",
         json_schema_extra=get_letta_message_content_union_str_json_schema(),
     )
-    name: Optional[str] = Field(None, description="The name of the participant.")
-    otid: Optional[str] = Field(None, description="The offline threading id associated with this message")
-    sender_id: Optional[str] = Field(None, description="The id of the sender of the message, can be an identity id or agent id")
-    batch_item_id: Optional[str] = Field(None, description="The id of the LLMBatchItem that this message is associated with")
-    group_id: Optional[str] = Field(None, description="The multi-agent group that the message was sent in")
+    name: Optional[str] = Field(default=None, description="The name of the participant.")
+    otid: Optional[str] = Field(default=None, description="The offline threading id associated with this message")
+    sender_id: Optional[str] = Field(default=None, description="The id of the sender of the message, can be an identity id or agent id")
+    batch_item_id: Optional[str] = Field(default=None, description="The id of the LLMBatchItem that this message is associated with")
+    group_id: Optional[str] = Field(default=None, description="The multi-agent group that the message was sent in")
 
     def model_dump(self, to_orm: bool = False, **kwargs) -> Dict[str, Any]:
         data = super().model_dump(**kwargs)
@@ -101,9 +101,9 @@ class MessageCreate(BaseModel):
 class MessageUpdate(BaseModel):
     """Request to update a message"""
 
-    role: Optional[MessageRole] = Field(None, description="The role of the participant.")
+    role: Optional[MessageRole] = Field(default=None, description="The role of the participant.")
     content: Optional[Union[str, List[LettaMessageContentUnion]]] = Field(
-        None,
+        default=None,
         description="The content of the message.",
         json_schema_extra=get_letta_message_content_union_str_json_schema(),
     )
@@ -112,11 +112,11 @@ class MessageUpdate(BaseModel):
     # agent_id: Optional[str] = Field(None, description="The unique identifier of the agent.")
     # NOTE: we probably shouldn't allow updating the model field, otherwise this loses meaning
     # model: Optional[str] = Field(None, description="The model used to make the function call.")
-    name: Optional[str] = Field(None, description="The name of the participant.")
+    name: Optional[str] = Field(default=None, description="The name of the participant.")
     # NOTE: we probably shouldn't allow updating the created_at field, right?
     # created_at: Optional[datetime] = Field(None, description="The time the message was created.")
-    tool_calls: Optional[List[OpenAIToolCall,]] = Field(None, description="The list of tool calls requested.")
-    tool_call_id: Optional[str] = Field(None, description="The id of the tool call.")
+    tool_calls: Optional[List[OpenAIToolCall,]] = Field(default=None, description="The list of tool calls requested.")
+    tool_call_id: Optional[str] = Field(default=None, description="The id of the tool call.")
 
     def model_dump(self, to_orm: bool = False, **kwargs) -> Dict[str, Any]:
         data = super().model_dump(**kwargs)
@@ -150,28 +150,28 @@ class Message(BaseMessage):
     """
 
     id: str = BaseMessage.generate_id_field()
-    organization_id: Optional[str] = Field(None, description="The unique identifier of the organization.")
-    agent_id: Optional[str] = Field(None, description="The unique identifier of the agent.")
-    model: Optional[str] = Field(None, description="The model used to make the function call.")
+    organization_id: Optional[str] = Field(default=None, description="The unique identifier of the organization.")
+    agent_id: Optional[str] = Field(default=None, description="The unique identifier of the agent.")
+    model: Optional[str] = Field(default=None, description="The model used to make the function call.")
     # Basic OpenAI-style fields
     role: MessageRole = Field(..., description="The role of the participant.")
-    content: Optional[List[LettaMessageContentUnion]] = Field(None, description="The content of the message.")
+    content: Optional[List[LettaMessageContentUnion]] = Field(default=None, description="The content of the message.")
     # NOTE: in OpenAI, this field is only used for roles 'user', 'assistant', and 'function' (now deprecated). 'tool' does not use it.
     name: Optional[str] = Field(
-        None,
+        default=None,
         description="For role user/assistant: the (optional) name of the participant. For role tool/function: the name of the function called.",
     )
     tool_calls: Optional[List[OpenAIToolCall]] = Field(
-        None, description="The list of tool calls requested. Only applicable for role assistant."
+        default=None, description="The list of tool calls requested. Only applicable for role assistant."
     )
-    tool_call_id: Optional[str] = Field(None, description="The ID of the tool call. Only applicable for role tool.")
+    tool_call_id: Optional[str] = Field(default=None, description="The ID of the tool call. Only applicable for role tool.")
     # Extras
-    step_id: Optional[str] = Field(None, description="The id of the step that this message was created in.")
-    otid: Optional[str] = Field(None, description="The offline threading id associated with this message")
-    tool_returns: Optional[List[ToolReturn]] = Field(None, description="Tool execution return information for prior tool calls")
-    group_id: Optional[str] = Field(None, description="The multi-agent group that the message was sent in")
-    sender_id: Optional[str] = Field(None, description="The id of the sender of the message, can be an identity id or agent id")
-    batch_item_id: Optional[str] = Field(None, description="The id of the LLMBatchItem that this message is associated with")
+    step_id: Optional[str] = Field(default=None, description="The id of the step that this message was created in.")
+    otid: Optional[str] = Field(default=None, description="The offline threading id associated with this message")
+    tool_returns: Optional[List[ToolReturn]] = Field(default=None, description="Tool execution return information for prior tool calls")
+    group_id: Optional[str] = Field(default=None, description="The multi-agent group that the message was sent in")
+    sender_id: Optional[str] = Field(default=None, description="The id of the sender of the message, can be an identity id or agent id")
+    batch_item_id: Optional[str] = Field(default=None, description="The id of the LLMBatchItem that this message is associated with")
     # This overrides the optional base orm schema, created_at MUST exist on all messages objects
     created_at: datetime = Field(default_factory=get_utc_time, description="The timestamp when the object was created.")
 
@@ -482,7 +482,9 @@ class Message(BaseMessage):
         # TODO(caren) implicit support for only non-parts/list content types
         if openai_message_dict["content"] is not None and type(openai_message_dict["content"]) is not str:
             raise ValueError(f"Invalid content type: {type(openai_message_dict['content'])}")
-        content = [TextContent(text=openai_message_dict["content"])] if openai_message_dict["content"] else []
+        content: List[LettaMessageContentUnion] = (
+            [TextContent(text=openai_message_dict["content"])] if openai_message_dict["content"] else []
+        )
 
         # TODO(caren) bad assumption here that "reasoning_content" always comes before "redacted_reasoning_content"
         if "reasoning_content" in openai_message_dict and openai_message_dict["reasoning_content"]:
@@ -491,14 +493,16 @@ class Message(BaseMessage):
                     reasoning=openai_message_dict["reasoning_content"],
                     is_native=True,
                     signature=(
-                        openai_message_dict["reasoning_content_signature"] if openai_message_dict["reasoning_content_signature"] else None
+                        str(openai_message_dict["reasoning_content_signature"])
+                        if "reasoning_content_signature" in openai_message_dict
+                        else None
                     ),
                 ),
             )
         if "redacted_reasoning_content" in openai_message_dict and openai_message_dict["redacted_reasoning_content"]:
             content.append(
                 RedactedReasoningContent(
-                    data=openai_message_dict["redacted_reasoning_content"] if "redacted_reasoning_content" in openai_message_dict else None,
+                    data=str(openai_message_dict["redacted_reasoning_content"]),
                 ),
             )
         if "omitted_reasoning_content" in openai_message_dict and openai_message_dict["omitted_reasoning_content"]:
@@ -694,7 +698,7 @@ class Message(BaseMessage):
         elif self.role == "assistant":
             assert self.tool_calls is not None or text_content is not None
             openai_message = {
-                "content": None if put_inner_thoughts_in_kwargs else text_content,
+                "content": None if (put_inner_thoughts_in_kwargs and self.tool_calls is not None) else text_content,
                 "role": self.role,
             }
 
@@ -733,7 +737,7 @@ class Message(BaseMessage):
             else:
                 warnings.warn(f"Using OpenAI with invalid 'name' field (name={self.name} role={self.role}).")
 
-        if parse_content_parts:
+        if parse_content_parts and self.content is not None:
             for content in self.content:
                 if isinstance(content, ReasoningContent):
                     openai_message["reasoning_content"] = content.reasoning
@@ -819,7 +823,7 @@ class Message(BaseMessage):
             }
             content = []
             # COT / reasoning / thinking
-            if len(self.content) > 1:
+            if self.content is not None and len(self.content) > 1:
                 for content_part in self.content:
                     if isinstance(content_part, ReasoningContent):
                         content.append(
@@ -1154,6 +1158,6 @@ class Message(BaseMessage):
 
 class ToolReturn(BaseModel):
     status: Literal["success", "error"] = Field(..., description="The status of the tool call")
-    stdout: Optional[List[str]] = Field(None, description="Captured stdout (e.g. prints, logs) from the tool invocation")
-    stderr: Optional[List[str]] = Field(None, description="Captured stderr from the tool invocation")
+    stdout: Optional[List[str]] = Field(default=None, description="Captured stdout (e.g. prints, logs) from the tool invocation")
+    stderr: Optional[List[str]] = Field(default=None, description="Captured stderr from the tool invocation")
     # func_return: Optional[Any] = Field(None, description="The function return object")

letta/schemas/providers.py

@@ -324,18 +324,25 @@ class OpenAIProvider(Provider):
             else:
                 handle = self.get_handle(model_name)
 
-            configs.append(
-                LLMConfig(
-                    model=model_name,
-                    model_endpoint_type="openai",
-                    model_endpoint=self.base_url,
-                    context_window=context_window_size,
-                    handle=handle,
-                    provider_name=self.name,
-                    provider_category=self.provider_category,
-                )
+            llm_config = LLMConfig(
+                model=model_name,
+                model_endpoint_type="openai",
+                model_endpoint=self.base_url,
+                context_window=context_window_size,
+                handle=handle,
+                provider_name=self.name,
+                provider_category=self.provider_category,
             )
 
+            # gpt-4o-mini has started to regress with pretty bad emoji spam loops
+            # this is to counteract that
+            if "gpt-4o-mini" in model_name:
+                llm_config.frequency_penalty = 1.0
+            if "gpt-4.1-mini" in model_name:
+                llm_config.frequency_penalty = 1.0
+
+            configs.append(llm_config)
+
         # for OpenAI, sort in reverse order
         if self.base_url == "https://api.openai.com/v1":
             # alphnumeric sort

letta/server/rest_api/app.py

@@ -17,6 +17,7 @@ from letta.__init__ import __version__ as letta_version
 from letta.agents.exceptions import IncompatibleAgentType
 from letta.constants import ADMIN_PREFIX, API_PREFIX, OPENAI_API_PREFIX
 from letta.errors import BedrockPermissionError, LettaAgentNotFoundError, LettaUserNotFoundError
+from letta.helpers.pinecone_utils import get_pinecone_indices, should_use_pinecone, upsert_pinecone_indices
 from letta.jobs.scheduler import start_scheduler_with_leader_election
 from letta.log import get_logger
 from letta.orm.errors import DatabaseTimeoutError, ForeignKeyConstraintViolationError, NoResultFound, UniqueConstraintViolationError
@@ -127,6 +128,16 @@ async def lifespan(app_: FastAPI):
     db_registry.initialize_async()
     logger.info(f"[Worker {worker_id}] Database connections initialized")
 
+    if should_use_pinecone():
+        if settings.upsert_pinecone_indices:
+            logger.info(f"[Worker {worker_id}] Upserting pinecone indices: {get_pinecone_indices()}")
+            await upsert_pinecone_indices()
+            logger.info(f"[Worker {worker_id}] Upserted pinecone indices")
+        else:
+            logger.info(f"[Worker {worker_id}] Enabled pinecone")
+    else:
+        logger.info(f"[Worker {worker_id}] Disabled pinecone")
+
     logger.info(f"[Worker {worker_id}] Starting scheduler with leader election")
     global server
     try:

letta/server/rest_api/routers/v1/agents.py

@@ -38,6 +38,7 @@ from letta.schemas.user import User
 from letta.serialize_schemas.pydantic_agent_schema import AgentSchema
 from letta.server.rest_api.utils import get_letta_server
 from letta.server.server import SyncServer
+from letta.services.summarizer.enums import SummarizationMode
 from letta.services.telemetry_manager import NoopTelemetryManager
 from letta.settings import settings
 from letta.utils import safe_create_task
@@ -750,6 +751,12 @@ async def send_message(
                     step_manager=server.step_manager,
                     telemetry_manager=server.telemetry_manager if settings.llm_api_logging else NoopTelemetryManager(),
                     current_run_id=run.id,
+                    # summarizer settings to be added here
+                    summarizer_mode=(
+                        SummarizationMode.STATIC_MESSAGE_BUFFER
+                        if agent.agent_type == AgentType.voice_convo_agent
+                        else SummarizationMode.PARTIAL_EVICT_MESSAGE_BUFFER
+                    ),
                 )
 
             result = await agent_loop.step(
@@ -878,6 +885,12 @@ async def send_message_streaming(
                     step_manager=server.step_manager,
                     telemetry_manager=server.telemetry_manager if settings.llm_api_logging else NoopTelemetryManager(),
                     current_run_id=run.id,
+                    # summarizer settings to be added here
+                    summarizer_mode=(
+                        SummarizationMode.STATIC_MESSAGE_BUFFER
+                        if agent.agent_type == AgentType.voice_convo_agent
+                        else SummarizationMode.PARTIAL_EVICT_MESSAGE_BUFFER
+                    ),
                 )
             from letta.server.rest_api.streaming_response import StreamingResponseWithStatusCode
 
@@ -1014,6 +1027,12 @@ async def _process_message_background(
                     actor=actor,
                     step_manager=server.step_manager,
                     telemetry_manager=server.telemetry_manager if settings.llm_api_logging else NoopTelemetryManager(),
+                    # summarizer settings to be added here
+                    summarizer_mode=(
+                        SummarizationMode.STATIC_MESSAGE_BUFFER
+                        if agent.agent_type == AgentType.voice_convo_agent
+                        else SummarizationMode.PARTIAL_EVICT_MESSAGE_BUFFER
+                    ),
                 )
 
             result = await agent_loop.step(

letta/server/rest_api/routers/v1/sources.py

@@ -9,6 +9,12 @@ from fastapi import APIRouter, Depends, Header, HTTPException, Query, UploadFile
 from starlette import status
 
 import letta.constants as constants
+from letta.helpers.pinecone_utils import (
+    delete_file_records_from_pinecone_index,
+    delete_source_records_from_pinecone_index,
+    list_pinecone_index_for_files,
+    should_use_pinecone,
+)
 from letta.log import get_logger
 from letta.otel.tracing import trace_method
 from letta.schemas.agent import AgentState
@@ -22,6 +28,7 @@ from letta.server.rest_api.utils import get_letta_server
 from letta.server.server import SyncServer
 from letta.services.file_processor.chunker.llama_index_chunker import LlamaIndexChunker
 from letta.services.file_processor.embedder.openai_embedder import OpenAIEmbedder
+from letta.services.file_processor.embedder.pinecone_embedder import PineconeEmbedder
 from letta.services.file_processor.file_processor import FileProcessor
 from letta.services.file_processor.file_types import (
     get_allowed_media_types,
@@ -163,6 +170,10 @@ async def delete_source(
     files = await server.file_manager.list_files(source_id, actor)
     file_ids = [f.id for f in files]
 
+    if should_use_pinecone():
+        logger.info(f"Deleting source {source_id} from pinecone index")
+        await delete_source_records_from_pinecone_index(source_id=source_id, actor=actor)
+
     for agent_state in agent_states:
         await server.remove_files_from_context_window(agent_state=agent_state, file_ids=file_ids, actor=actor)
 
@@ -326,16 +337,24 @@ async def get_file_metadata(
     """
     actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)
 
-    # Verify the source exists and user has access
-    source = await server.source_manager.get_source_by_id(source_id=source_id, actor=actor)
-    if not source:
-        raise HTTPException(status_code=404, detail=f"Source with id={source_id} not found.")
-
     # Get file metadata using the file manager
     file_metadata = await server.file_manager.get_file_by_id(
         file_id=file_id, actor=actor, include_content=include_content, strip_directory_prefix=True
     )
 
+    if should_use_pinecone() and not file_metadata.is_processing_terminal():
+        ids = await list_pinecone_index_for_files(file_id=file_id, actor=actor, limit=file_metadata.total_chunks)
+        logger.info(f"Embedded chunks {len(ids)}/{file_metadata.total_chunks} for {file_id} in organization {actor.organization_id}")
+
+        if len(ids) != file_metadata.chunks_embedded or len(ids) == file_metadata.total_chunks:
+            if len(ids) != file_metadata.total_chunks:
+                file_status = file_metadata.processing_status
+            else:
+                file_status = FileProcessingStatus.COMPLETED
+            await server.file_manager.update_file_status(
+                file_id=file_metadata.id, actor=actor, chunks_embedded=len(ids), processing_status=file_status
+            )
+
     if not file_metadata:
         raise HTTPException(status_code=404, detail=f"File with id={file_id} not found.")
 
@@ -364,6 +383,10 @@ async def delete_file_from_source(
 
     await server.remove_file_from_context_windows(source_id=source_id, file_id=deleted_file.id, actor=actor)
 
+    if should_use_pinecone():
+        logger.info(f"Deleting file {file_id} from pinecone index")
+        await delete_file_records_from_pinecone_index(file_id=file_id, actor=actor)
+
     asyncio.create_task(sleeptime_document_ingest_async(server, source_id, actor, clear_history=True))
     if deleted_file is None:
         raise HTTPException(status_code=404, detail=f"File with id={file_id} not found.")
@@ -402,8 +425,14 @@ async def load_file_to_source_cloud(
 ):
     file_processor = MistralFileParser()
     text_chunker = LlamaIndexChunker(chunk_size=embedding_config.embedding_chunk_size)
-    embedder = OpenAIEmbedder(embedding_config=embedding_config)
-    file_processor = FileProcessor(file_parser=file_processor, text_chunker=text_chunker, embedder=embedder, actor=actor)
+    using_pinecone = should_use_pinecone()
+    if using_pinecone:
+        embedder = PineconeEmbedder()
+    else:
+        embedder = OpenAIEmbedder(embedding_config=embedding_config)
+    file_processor = FileProcessor(
+        file_parser=file_processor, text_chunker=text_chunker, embedder=embedder, actor=actor, using_pinecone=using_pinecone
+    )
     await file_processor.process(
         server=server, agent_states=agent_states, source_id=source_id, content=content, file_metadata=file_metadata
     )

letta/services/file_manager.py

@@ -109,15 +109,17 @@ class FileManager:
         actor: PydanticUser,
         processing_status: Optional[FileProcessingStatus] = None,
         error_message: Optional[str] = None,
+        total_chunks: Optional[int] = None,
+        chunks_embedded: Optional[int] = None,
     ) -> PydanticFileMetadata:
         """
-        Update processing_status and/or error_message on a FileMetadata row.
+        Update processing_status, error_message, total_chunks, and/or chunks_embedded on a FileMetadata row.
 
         * 1st round-trip → UPDATE
         * 2nd round-trip → SELECT fresh row (same as read_async)
         """
 
-        if processing_status is None and error_message is None:
+        if processing_status is None and error_message is None and total_chunks is None and chunks_embedded is None:
             raise ValueError("Nothing to update")
 
         values: dict[str, object] = {"updated_at": datetime.utcnow()}
@@ -125,6 +127,10 @@ class FileManager:
             values["processing_status"] = processing_status
         if error_message is not None:
             values["error_message"] = error_message
+        if total_chunks is not None:
+            values["total_chunks"] = total_chunks
+        if chunks_embedded is not None:
+            values["chunks_embedded"] = chunks_embedded
 
         async with db_registry.async_session() as session:
             # Fast in-place update – no ORM hydration

letta/services/file_processor/embedder/base_embedder.py

@@ -0,0 +1,16 @@
+from abc import ABC, abstractmethod
+from typing import List
+
+from letta.log import get_logger
+from letta.schemas.passage import Passage
+from letta.schemas.user import User
+
+logger = get_logger(__name__)
+
+
+class BaseEmbedder(ABC):
+    """Abstract base class for embedding generation"""
+
+    @abstractmethod
+    async def generate_embedded_passages(self, file_id: str, source_id: str, chunks: List[str], actor: User) -> List[Passage]:
+        """Generate embeddings for chunks with batching and concurrent processing"""

letta/services/file_processor/embedder/openai_embedder.py

@@ -9,12 +9,13 @@ from letta.schemas.embedding_config import EmbeddingConfig
 from letta.schemas.enums import ProviderType
 from letta.schemas.passage import Passage
 from letta.schemas.user import User
+from letta.services.file_processor.embedder.base_embedder import BaseEmbedder
 from letta.settings import model_settings
 
 logger = get_logger(__name__)
 
 
-class OpenAIEmbedder:
+class OpenAIEmbedder(BaseEmbedder):
     """OpenAI-based embedding generation"""
 
     def __init__(self, embedding_config: Optional[EmbeddingConfig] = None):
@@ -24,6 +25,7 @@ class OpenAIEmbedder:
             else EmbeddingConfig.default_config(model_name="letta")
         )
         self.embedding_config = embedding_config or self.default_embedding_config
+        self.max_concurrent_requests = 20
 
         # TODO: Unify to global OpenAI client
         self.client: OpenAIClient = cast(
@@ -34,7 +36,6 @@ class OpenAIEmbedder:
                 actor=None,  # Not necessary
             ),
         )
-        self.max_concurrent_requests = 20
 
     @trace_method
     async def _embed_batch(self, batch: List[str], batch_indices: List[int]) -> List[Tuple[int, List[float]]]:

letta/services/file_processor/embedder/pinecone_embedder.py

@@ -0,0 +1,74 @@
+from typing import List
+
+from letta.helpers.pinecone_utils import upsert_file_records_to_pinecone_index
+from letta.log import get_logger
+from letta.otel.tracing import log_event, trace_method
+from letta.schemas.passage import Passage
+from letta.schemas.user import User
+from letta.services.file_processor.embedder.base_embedder import BaseEmbedder
+
+try:
+    PINECONE_AVAILABLE = True
+except ImportError:
+    PINECONE_AVAILABLE = False
+
+logger = get_logger(__name__)
+
+
+class PineconeEmbedder(BaseEmbedder):
+    """Pinecone-based embedding generation"""
+
+    def __init__(self):
+        if not PINECONE_AVAILABLE:
+            raise ImportError("Pinecone package is not installed. Install it with: pip install pinecone")
+
+        super().__init__()
+
+    @trace_method
+    async def generate_embedded_passages(self, file_id: str, source_id: str, chunks: List[str], actor: User) -> List[Passage]:
+        """Generate embeddings and upsert to Pinecone, then return Passage objects"""
+        if not chunks:
+            return []
+
+        logger.info(f"Upserting {len(chunks)} chunks to Pinecone using namespace {source_id}")
+        log_event(
+            "embedder.generation_started",
+            {
+                "total_chunks": len(chunks),
+                "file_id": file_id,
+                "source_id": source_id,
+            },
+        )
+
+        # Upsert records to Pinecone using source_id as namespace
+        try:
+            await upsert_file_records_to_pinecone_index(file_id=file_id, source_id=source_id, chunks=chunks, actor=actor)
+            logger.info(f"Successfully kicked off upserting {len(chunks)} records to Pinecone")
+            log_event(
+                "embedder.upsert_started",
+                {"records_upserted": len(chunks), "namespace": source_id, "file_id": file_id},
+            )
+        except Exception as e:
+            logger.error(f"Failed to upsert records to Pinecone: {str(e)}")
+            log_event("embedder.upsert_failed", {"error": str(e), "error_type": type(e).__name__})
+            raise
+
+        # Create Passage objects (without embeddings since Pinecone handles them)
+        passages = []
+        for i, text in enumerate(chunks):
+            passage = Passage(
+                text=text,
+                file_id=file_id,
+                source_id=source_id,
+                embedding=None,  # Pinecone handles embeddings internally
+                embedding_config=None,  # None
+                organization_id=actor.organization_id,
+            )
+            passages.append(passage)
+
+        logger.info(f"Successfully created {len(passages)} passages")
+        log_event(
+            "embedder.generation_completed",
+            {"passages_created": len(passages), "total_chunks_processed": len(chunks), "file_id": file_id, "source_id": source_id},
+        )
+        return passages

letta/services/file_processor/file_processor.py

@@ -11,7 +11,7 @@ from letta.server.server import SyncServer
 from letta.services.file_manager import FileManager
 from letta.services.file_processor.chunker.line_chunker import LineChunker
 from letta.services.file_processor.chunker.llama_index_chunker import LlamaIndexChunker
-from letta.services.file_processor.embedder.openai_embedder import OpenAIEmbedder
+from letta.services.file_processor.embedder.base_embedder import BaseEmbedder
 from letta.services.file_processor.parser.mistral_parser import MistralFileParser
 from letta.services.job_manager import JobManager
 from letta.services.passage_manager import PassageManager
@@ -27,8 +27,9 @@ class FileProcessor:
         self,
         file_parser: MistralFileParser,
         text_chunker: LlamaIndexChunker,
-        embedder: OpenAIEmbedder,
+        embedder: BaseEmbedder,
         actor: User,
+        using_pinecone: bool,
         max_file_size: int = 50 * 1024 * 1024,  # 50MB default
     ):
         self.file_parser = file_parser
@@ -41,6 +42,7 @@ class FileProcessor:
         self.passage_manager = PassageManager()
         self.job_manager = JobManager()
         self.actor = actor
+        self.using_pinecone = using_pinecone
 
     # TODO: Factor this function out of SyncServer
     @trace_method
@@ -109,7 +111,7 @@ class FileProcessor:
 
             logger.info("Chunking extracted text")
             log_event("file_processor.chunking_started", {"filename": filename, "pages_to_process": len(ocr_response.pages)})
-            all_passages = []
+            all_chunks = []
 
             for page in ocr_response.pages:
                 chunks = self.text_chunker.chunk_text(page)
@@ -118,24 +120,17 @@ class FileProcessor:
                     log_event("file_processor.chunking_failed", {"filename": filename, "page_index": ocr_response.pages.index(page)})
                     raise ValueError("No chunks created from text")
 
-                passages = await self.embedder.generate_embedded_passages(
-                    file_id=file_metadata.id, source_id=source_id, chunks=chunks, actor=self.actor
-                )
-                log_event(
-                    "file_processor.page_processed",
-                    {
-                        "filename": filename,
-                        "page_index": ocr_response.pages.index(page),
-                        "chunks_created": len(chunks),
-                        "passages_generated": len(passages),
-                    },
-                )
-                all_passages.extend(passages)
+                all_chunks.extend(self.text_chunker.chunk_text(page))
 
-            all_passages = await self.passage_manager.create_many_source_passages_async(
-                passages=all_passages, file_metadata=file_metadata, actor=self.actor
+            all_passages = await self.embedder.generate_embedded_passages(
+                file_id=file_metadata.id, source_id=source_id, chunks=all_chunks, actor=self.actor
             )
-            log_event("file_processor.passages_created", {"filename": filename, "total_passages": len(all_passages)})
+
+            if not self.using_pinecone:
+                all_passages = await self.passage_manager.create_many_source_passages_async(
+                    passages=all_passages, file_metadata=file_metadata, actor=self.actor
+                )
+                log_event("file_processor.passages_created", {"filename": filename, "total_passages": len(all_passages)})
 
             logger.info(f"Successfully processed {filename}: {len(all_passages)} passages")
             log_event(
@@ -149,9 +144,14 @@ class FileProcessor:
             )
 
             # update job status
-            await self.file_manager.update_file_status(
-                file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.COMPLETED
-            )
+            if not self.using_pinecone:
+                await self.file_manager.update_file_status(
+                    file_id=file_metadata.id, actor=self.actor, processing_status=FileProcessingStatus.COMPLETED
+                )
+            else:
+                await self.file_manager.update_file_status(
+                    file_id=file_metadata.id, actor=self.actor, total_chunks=len(all_passages), chunks_embedded=0
+                )
 
             return all_passages
 

letta/services/job_manager.py

@@ -115,10 +115,6 @@ class JobManager:
                 job.completed_at = get_utc_time().replace(tzinfo=None)
                 if job.callback_url:
                     await self._dispatch_callback_async(job)
-                else:
-                    logger.info(f"Job does not contain callback url: {job}")
-            else:
-                logger.info(f"Job update is not terminal {job_update}")
 
             # Save the updated job to the database
             await job.update_async(db_session=session, actor=actor)

letta/services/source_manager.py

@@ -19,7 +19,6 @@ class SourceManager:
     @trace_method
     async def create_source(self, source: PydanticSource, actor: PydanticUser) -> PydanticSource:
         """Create a new source based on the PydanticSource schema."""
-        # Try getting the source first by id
         db_source = await self.get_source_by_id(source.id, actor=actor)
         if db_source:
             return db_source

letta/services/summarizer/enums.py

@@ -7,3 +7,4 @@ class SummarizationMode(str, Enum):
     """
 
     STATIC_MESSAGE_BUFFER = "static_message_buffer_mode"
+    PARTIAL_EVICT_MESSAGE_BUFFER = "partial_evict_message_buffer_mode"