letta-nightly 0.11.3.dev20250819104229__tar.gz → 0.11.4.dev20250820213507__tar.gz

{letta_nightly-0.11.3.dev20250819104229 → letta_nightly-0.11.4.dev20250820213507}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: letta-nightly
-Version: 0.11.3.dev20250819104229
+Version: 0.11.4.dev20250820213507
 Summary: Create LLM agents with long-term memory and custom tools
 License: Apache License
 Author: Letta Team
@@ -68,7 +68,7 @@ Requires-Dist: mistralai (>=1.8.1,<2.0.0)
 Requires-Dist: modal (>=1.1.0,<2.0.0) ; extra == "cloud-tool-sandbox"
 Requires-Dist: nltk (>=3.8.1,<4.0.0)
 Requires-Dist: numpy (>=2.1.0,<3.0.0)
-Requires-Dist: openai (>=1.60.0,<2.0.0)
+Requires-Dist: openai (>=1.99.9,<2.0.0)
 Requires-Dist: opentelemetry-api (==1.30.0)
 Requires-Dist: opentelemetry-exporter-otlp (==1.30.0)
 Requires-Dist: opentelemetry-instrumentation-requests (==0.51b0)

{letta_nightly-0.11.3.dev20250819104229 → letta_nightly-0.11.4.dev20250820213507}/letta/__init__.py

@@ -5,7 +5,7 @@ try:
     __version__ = version("letta")
 except PackageNotFoundError:
     # Fallback for development installations
-    __version__ = "0.11.3"
+    __version__ = "0.11.4"
 
 if os.environ.get("LETTA_VERSION"):
     __version__ = os.environ["LETTA_VERSION"]

{letta_nightly-0.11.3.dev20250819104229 → letta_nightly-0.11.4.dev20250820213507}/letta/agents/helpers.py

@@ -4,6 +4,7 @@ import xml.etree.ElementTree as ET
 from typing import List, Optional, Tuple
 
 from letta.helpers import ToolRulesSolver
+from letta.log import get_logger
 from letta.schemas.agent import AgentState
 from letta.schemas.letta_message import MessageType
 from letta.schemas.letta_response import LettaResponse
@@ -15,6 +16,8 @@ from letta.schemas.user import User
 from letta.server.rest_api.utils import create_input_messages
 from letta.services.message_manager import MessageManager
 
+logger = get_logger(__name__)
+
 
 def _create_letta_response(
     new_in_context_messages: list[Message],
@@ -222,6 +225,7 @@ def _safe_load_tool_call_str(tool_call_args_str: str) -> dict:
             # Load it again - this is due to sometimes Anthropic returning weird json @caren
             tool_args = json.loads(tool_args)
     except json.JSONDecodeError:
+        logger.error("Failed to JSON decode tool call argument string: %s", tool_call_args_str)
         tool_args = {}
 
     return tool_args

{letta_nightly-0.11.3.dev20250819104229 → letta_nightly-0.11.4.dev20250820213507}/letta/agents/letta_agent.py

@@ -44,6 +44,7 @@ from letta.schemas.message import Message, MessageCreate
 from letta.schemas.openai.chat_completion_response import ToolCall, UsageStatistics
 from letta.schemas.provider_trace import ProviderTraceCreate
 from letta.schemas.step import StepProgression
+from letta.schemas.step_metrics import StepMetrics
 from letta.schemas.tool_execution_result import ToolExecutionResult
 from letta.schemas.usage import LettaUsageStatistics
 from letta.schemas.user import User
@@ -242,6 +243,7 @@ class LettaAgent(BaseAgent):
 
             step_progression = StepProgression.START
             should_continue = False
+            step_metrics = StepMetrics(id=step_id)  # Initialize metrics tracking
 
             # Create step early with PENDING status
             logged_step = await self.step_manager.log_step_async(
@@ -271,6 +273,7 @@ class LettaAgent(BaseAgent):
                         llm_client,
                         tool_rules_solver,
                         agent_step_span,
+                        step_metrics,
                     )
                 )
                 in_context_messages = current_in_context_messages + new_in_context_messages
@@ -320,6 +323,7 @@ class LettaAgent(BaseAgent):
                     initial_messages=initial_messages,
                     agent_step_span=agent_step_span,
                     is_final_step=(i == max_steps - 1),
+                    step_metrics=step_metrics,
                 )
                 step_progression = StepProgression.STEP_LOGGED
 
@@ -365,6 +369,17 @@ class LettaAgent(BaseAgent):
 
                 MetricRegistry().step_execution_time_ms_histogram.record(get_utc_timestamp_ns() - step_start, get_ctx_attributes())
                 step_progression = StepProgression.FINISHED
+
+                # Record step metrics for successful completion
+                if logged_step and step_metrics:
+                    # Set the step_ns that was already calculated
+                    step_metrics.step_ns = step_ns
+                    await self._record_step_metrics(
+                        step_id=step_id,
+                        agent_state=agent_state,
+                        step_metrics=step_metrics,
+                    )
+
             except Exception as e:
                 # Handle any unexpected errors during step processing
                 self.logger.error(f"Error during step processing: {e}")
@@ -416,7 +431,7 @@ class LettaAgent(BaseAgent):
 
                     if step_progression <= StepProgression.RESPONSE_RECEIVED:
                         # TODO (cliandy): persist response if we get it back
-                        if settings.track_errored_messages:
+                        if settings.track_errored_messages and initial_messages:
                             for message in initial_messages:
                                 message.is_err = True
                                 message.step_id = effective_step_id
@@ -433,6 +448,17 @@ class LettaAgent(BaseAgent):
                     if settings.track_stop_reason:
                         await self._log_request(request_start_timestamp_ns, request_span, job_update_metadata, is_error=True)
 
+                    # Record partial step metrics on failure (capture whatever timing data we have)
+                    if logged_step and step_metrics and step_progression < StepProgression.FINISHED:
+                        # Calculate total step time up to the failure point
+                        step_metrics.step_ns = get_utc_timestamp_ns() - step_start
+                        await self._record_step_metrics(
+                            step_id=step_id,
+                            agent_state=agent_state,
+                            step_metrics=step_metrics,
+                            job_id=locals().get("run_id", self.current_run_id),
+                        )
+
                 except Exception as e:
                     self.logger.error("Failed to update step: %s", e)
 
@@ -513,6 +539,7 @@ class LettaAgent(BaseAgent):
 
             step_progression = StepProgression.START
             should_continue = False
+            step_metrics = StepMetrics(id=step_id)  # Initialize metrics tracking
 
             # Create step early with PENDING status
             logged_step = await self.step_manager.log_step_async(
@@ -536,7 +563,13 @@ class LettaAgent(BaseAgent):
             try:
                 request_data, response_data, current_in_context_messages, new_in_context_messages, valid_tool_names = (
                     await self._build_and_request_from_llm(
-                        current_in_context_messages, new_in_context_messages, agent_state, llm_client, tool_rules_solver, agent_step_span
+                        current_in_context_messages,
+                        new_in_context_messages,
+                        agent_state,
+                        llm_client,
+                        tool_rules_solver,
+                        agent_step_span,
+                        step_metrics,
                     )
                 )
                 in_context_messages = current_in_context_messages + new_in_context_messages
@@ -587,6 +620,7 @@ class LettaAgent(BaseAgent):
                     agent_step_span=agent_step_span,
                     is_final_step=(i == max_steps - 1),
                     run_id=run_id,
+                    step_metrics=step_metrics,
                 )
                 step_progression = StepProgression.STEP_LOGGED
 
@@ -622,6 +656,17 @@ class LettaAgent(BaseAgent):
                 MetricRegistry().step_execution_time_ms_histogram.record(get_utc_timestamp_ns() - step_start, get_ctx_attributes())
                 step_progression = StepProgression.FINISHED
 
+                # Record step metrics for successful completion
+                if logged_step and step_metrics:
+                    # Set the step_ns that was already calculated
+                    step_metrics.step_ns = step_ns
+                    await self._record_step_metrics(
+                        step_id=step_id,
+                        agent_state=agent_state,
+                        step_metrics=step_metrics,
+                        job_id=run_id if run_id else self.current_run_id,
+                    )
+
             except Exception as e:
                 # Handle any unexpected errors during step processing
                 self.logger.error(f"Error during step processing: {e}")
@@ -669,7 +714,7 @@ class LettaAgent(BaseAgent):
 
                     if step_progression <= StepProgression.RESPONSE_RECEIVED:
                         # TODO (cliandy): persist response if we get it back
-                        if settings.track_errored_messages:
+                        if settings.track_errored_messages and initial_messages:
                             for message in initial_messages:
                                 message.is_err = True
                                 message.step_id = effective_step_id
@@ -686,6 +731,17 @@ class LettaAgent(BaseAgent):
                     if settings.track_stop_reason:
                         await self._log_request(request_start_timestamp_ns, request_span, job_update_metadata, is_error=True)
 
+                    # Record partial step metrics on failure (capture whatever timing data we have)
+                    if logged_step and step_metrics and step_progression < StepProgression.FINISHED:
+                        # Calculate total step time up to the failure point
+                        step_metrics.step_ns = get_utc_timestamp_ns() - step_start
+                        await self._record_step_metrics(
+                            step_id=step_id,
+                            agent_state=agent_state,
+                            step_metrics=step_metrics,
+                            job_id=locals().get("run_id", self.current_run_id),
+                        )
+
                 except Exception as e:
                     self.logger.error("Failed to update step: %s", e)
 
@@ -774,6 +830,7 @@ class LettaAgent(BaseAgent):
 
             step_progression = StepProgression.START
             should_continue = False
+            step_metrics = StepMetrics(id=step_id)  # Initialize metrics tracking
 
             # Create step early with PENDING status
             logged_step = await self.step_manager.log_step_async(
@@ -875,7 +932,10 @@ class LettaAgent(BaseAgent):
                 )
 
                 # log LLM request time
-                llm_request_ms = ns_to_ms(stream_end_time_ns - provider_request_start_timestamp_ns)
+                llm_request_ns = stream_end_time_ns - provider_request_start_timestamp_ns
+                step_metrics.llm_request_ns = llm_request_ns
+
+                llm_request_ms = ns_to_ms(llm_request_ns)
                 agent_step_span.add_event(name="llm_request_ms", attributes={"duration_ms": llm_request_ms})
                 MetricRegistry().llm_execution_time_ms_histogram.record(
                     llm_request_ms,
@@ -908,6 +968,7 @@ class LettaAgent(BaseAgent):
                     initial_messages=initial_messages,
                     agent_step_span=agent_step_span,
                     is_final_step=(i == max_steps - 1),
+                    step_metrics=step_metrics,
                 )
                 step_progression = StepProgression.STEP_LOGGED
 
@@ -979,6 +1040,25 @@ class LettaAgent(BaseAgent):
                 MetricRegistry().step_execution_time_ms_histogram.record(get_utc_timestamp_ns() - step_start, get_ctx_attributes())
                 step_progression = StepProgression.FINISHED
 
+                # Record step metrics for successful completion
+                if logged_step and step_metrics:
+                    try:
+                        # Set the step_ns that was already calculated
+                        step_metrics.step_ns = step_ns
+
+                        # Get context attributes for project and template IDs
+                        ctx_attrs = get_ctx_attributes()
+
+                        await self._record_step_metrics(
+                            step_id=step_id,
+                            agent_state=agent_state,
+                            step_metrics=step_metrics,
+                            ctx_attrs=ctx_attrs,
+                            job_id=self.current_run_id,
+                        )
+                    except Exception as metrics_error:
+                        self.logger.warning(f"Failed to record step metrics: {metrics_error}")
+
             except Exception as e:
                 # Handle any unexpected errors during step processing
                 self.logger.error(f"Error during step processing: {e}")
@@ -1029,7 +1109,7 @@ class LettaAgent(BaseAgent):
                             )
 
                     if step_progression <= StepProgression.STREAM_RECEIVED:
-                        if first_chunk and settings.track_errored_messages:
+                        if first_chunk and settings.track_errored_messages and initial_messages:
                             for message in initial_messages:
                                 message.is_err = True
                                 message.step_id = effective_step_id
@@ -1047,6 +1127,25 @@ class LettaAgent(BaseAgent):
                     if settings.track_stop_reason:
                         await self._log_request(request_start_timestamp_ns, request_span, job_update_metadata, is_error=True)
 
+                    # Record partial step metrics on failure (capture whatever timing data we have)
+                    if logged_step and step_metrics and step_progression < StepProgression.FINISHED:
+                        try:
+                            # Calculate total step time up to the failure point
+                            step_metrics.step_ns = get_utc_timestamp_ns() - step_start
+
+                            # Get context attributes for project and template IDs
+                            ctx_attrs = get_ctx_attributes()
+
+                            await self._record_step_metrics(
+                                step_id=step_id,
+                                agent_state=agent_state,
+                                step_metrics=step_metrics,
+                                ctx_attrs=ctx_attrs,
+                                job_id=locals().get("run_id", self.current_run_id),
+                            )
+                        except Exception as metrics_error:
+                            self.logger.warning(f"Failed to record step metrics: {metrics_error}")
+
                 except Exception as e:
                     self.logger.error("Failed to update step: %s", e)
 
@@ -1087,6 +1186,32 @@ class LettaAgent(BaseAgent):
         if request_span:
             request_span.end()
 
+    async def _record_step_metrics(
+        self,
+        *,
+        step_id: str,
+        agent_state: AgentState,
+        step_metrics: StepMetrics,
+        ctx_attrs: dict | None = None,
+        job_id: str | None = None,
+    ) -> None:
+        try:
+            attrs = ctx_attrs or get_ctx_attributes()
+            await self.step_manager.record_step_metrics_async(
+                actor=self.actor,
+                step_id=step_id,
+                llm_request_ns=step_metrics.llm_request_ns,
+                tool_execution_ns=step_metrics.tool_execution_ns,
+                step_ns=step_metrics.step_ns,
+                agent_id=agent_state.id,
+                job_id=job_id or self.current_run_id,
+                project_id=attrs.get("project.id") or agent_state.project_id,
+                template_id=attrs.get("template.id"),
+                base_template_id=attrs.get("base_template.id"),
+            )
+        except Exception as metrics_error:
+            self.logger.warning(f"Failed to record step metrics: {metrics_error}")
+
     # noinspection PyInconsistentReturns
     async def _build_and_request_from_llm(
         self,
@@ -1096,6 +1221,7 @@ class LettaAgent(BaseAgent):
         llm_client: LLMClientBase,
         tool_rules_solver: ToolRulesSolver,
         agent_step_span: "Span",
+        step_metrics: StepMetrics,
     ) -> tuple[dict, dict, list[Message], list[Message], list[str]] | None:
         for attempt in range(self.max_summarization_retries + 1):
             try:
@@ -1112,6 +1238,10 @@ class LettaAgent(BaseAgent):
                 async with AsyncTimer() as timer:
                     # Attempt LLM request
                     response = await llm_client.request_async(request_data, agent_state.llm_config)
+
+                # Track LLM request time
+                step_metrics.llm_request_ns = int(timer.elapsed_ns)
+
                 MetricRegistry().llm_execution_time_ms_histogram.record(
                     timer.elapsed_ms,
                     dict(get_ctx_attributes(), **{"model.name": agent_state.llm_config.model}),
@@ -1352,6 +1482,7 @@ class LettaAgent(BaseAgent):
         agent_step_span: Optional["Span"] = None,
         is_final_step: bool | None = None,
         run_id: str | None = None,
+        step_metrics: StepMetrics = None,
     ) -> tuple[list[Message], bool, LettaStopReason | None]:
         """
         Handle the final AI response once streaming completes, execute / validate the
@@ -1378,6 +1509,8 @@ class LettaAgent(BaseAgent):
         if tool_rule_violated:
             tool_execution_result = _build_rule_violation_result(tool_call_name, valid_tool_names, tool_rules_solver)
         else:
+            # Track tool execution time
+            tool_start_time = get_utc_timestamp_ns()
             tool_execution_result = await self._execute_tool(
                 tool_name=tool_call_name,
                 tool_args=tool_args,
@@ -1385,6 +1518,10 @@ class LettaAgent(BaseAgent):
                 agent_step_span=agent_step_span,
                 step_id=step_id,
             )
+            tool_end_time = get_utc_timestamp_ns()
+
+            # Store tool execution time in metrics
+            step_metrics.tool_execution_ns = tool_end_time - tool_start_time
 
         log_telemetry(
             self.logger, "_handle_ai_response execute tool finish", tool_execution_result=tool_execution_result, tool_call_id=tool_call_id

{letta_nightly-0.11.3.dev20250819104229 → letta_nightly-0.11.4.dev20250820213507}/letta/constants.py

@@ -36,6 +36,7 @@ LETTA_TOOL_MODULE_NAMES = [
 DEFAULT_ORG_ID = "org-00000000-0000-4000-8000-000000000000"
 DEFAULT_ORG_NAME = "default_org"
 
+AGENT_ID_PATTERN = re.compile(r"^agent-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$", re.IGNORECASE)
 
 # String in the error message for when the context window is too large
 # Example full message:
@@ -330,15 +331,15 @@ READ_ONLY_BLOCK_EDIT_ERROR = f"{ERROR_MESSAGE_PREFIX} This block is read-only an
 MESSAGE_SUMMARY_REQUEST_ACK = "Understood, I will respond with a summary of the message (and only the summary, nothing else) once I receive the conversation history. I'm ready."
 
 # Maximum length of an error message
-MAX_ERROR_MESSAGE_CHAR_LIMIT = 500
+MAX_ERROR_MESSAGE_CHAR_LIMIT = 1000
 
 # Default memory limits
-CORE_MEMORY_PERSONA_CHAR_LIMIT: int = 5000
-CORE_MEMORY_HUMAN_CHAR_LIMIT: int = 5000
-CORE_MEMORY_BLOCK_CHAR_LIMIT: int = 5000
+CORE_MEMORY_PERSONA_CHAR_LIMIT: int = 20000
+CORE_MEMORY_HUMAN_CHAR_LIMIT: int = 20000
+CORE_MEMORY_BLOCK_CHAR_LIMIT: int = 20000
 
 # Function return limits
-FUNCTION_RETURN_CHAR_LIMIT = 6000  # ~300 words
+FUNCTION_RETURN_CHAR_LIMIT = 50000  # ~300 words
 BASE_FUNCTION_RETURN_CHAR_LIMIT = 1000000  # very high (we rely on implementation)
 FILE_IS_TRUNCATED_WARNING = "# NOTE: This block is truncated, use functions to view the full content."
 
@@ -396,5 +397,7 @@ PINECONE_THROTTLE_DELAY = 0.75  # seconds base delay between batches
 WEB_SEARCH_MODEL_ENV_VAR_NAME = "LETTA_BUILTIN_WEBSEARCH_OPENAI_MODEL_NAME"
 WEB_SEARCH_MODEL_ENV_VAR_DEFAULT_VALUE = "gpt-4.1-mini-2025-04-14"
 
-# Excluded providers from base tool rules
-EXCLUDED_PROVIDERS_FROM_BASE_TOOL_RULES = {"anthropic", "openai", "google_ai", "google_vertex"}
+# Excluded model keywords from base tool rules
+EXCLUDE_MODEL_KEYWORDS_FROM_BASE_TOOL_RULES = ["claude-4-sonnet", "claude-3-5-sonnet", "gpt-5", "gemini-2.5-pro"]
+# But include models with these keywords in base tool rules (overrides exclusion)
+INCLUDE_MODEL_KEYWORDS_BASE_TOOL_RULES = ["mini"]

{letta_nightly-0.11.3.dev20250819104229 → letta_nightly-0.11.4.dev20250820213507}/letta/data_sources/connectors.py

@@ -4,7 +4,6 @@ import typer
 
 from letta.constants import EMBEDDING_BATCH_SIZE
 from letta.data_sources.connectors_helper import assert_all_files_exist_locally, extract_metadata_from_files, get_filenames_in_dir
-from letta.embeddings import embedding_model
 from letta.schemas.file import FileMetadata
 from letta.schemas.passage import Passage
 from letta.schemas.source import Source
@@ -40,61 +39,29 @@ class DataConnector:
 
 async def load_data(connector: DataConnector, source: Source, passage_manager: PassageManager, file_manager: FileManager, actor: "User"):
     from letta.llm_api.llm_client import LLMClient
-    from letta.schemas.embedding_config import EmbeddingConfig
 
     """Load data from a connector (generates file and passages) into a specified source_id, associated with a user_id."""
     embedding_config = source.embedding_config
 
     # insert passages/file
-    texts = []
     embedding_to_document_name = {}
     passage_count = 0
     file_count = 0
 
-    async def generate_embeddings(texts: List[str], embedding_config: EmbeddingConfig) -> List[Passage]:
-        passages = []
-        if embedding_config.embedding_endpoint_type == "openai":
-            texts.append(passage_text)
-
-            client = LLMClient.create(
-                provider_type=embedding_config.embedding_endpoint_type,
-                actor=actor,
-            )
-            embeddings = await client.request_embeddings(texts, embedding_config)
-
-        else:
-            embed_model = embedding_model(embedding_config)
-            embeddings = [embed_model.get_text_embedding(text) for text in texts]
-
-        # collate passage and embedding
-        for text, embedding in zip(texts, embeddings):
-            passage = Passage(
-                text=text,
-                file_id=file_metadata.id,
-                source_id=source.id,
-                metadata=passage_metadata,
-                organization_id=source.organization_id,
-                embedding_config=source.embedding_config,
-                embedding=embedding,
-            )
-            hashable_embedding = tuple(passage.embedding)
-            file_name = file_metadata.file_name
-            if hashable_embedding in embedding_to_document_name:
-                typer.secho(
-                    f"Warning: Duplicate embedding found for passage in {file_name} (already exists in {embedding_to_document_name[hashable_embedding]}), skipping insert into VectorDB.",
-                    fg=typer.colors.YELLOW,
-                )
-                continue
-
-            passages.append(passage)
-            embedding_to_document_name[hashable_embedding] = file_name
-        return passages
+    # Use the new LLMClient for all embedding requests
+    client = LLMClient.create(
+        provider_type=embedding_config.embedding_endpoint_type,
+        actor=actor,
+    )
 
     for file_metadata in connector.find_files(source):
         file_count += 1
         await file_manager.create_file(file_metadata, actor)
 
-        # generate passages
+        # generate passages for this file
+        texts = []
+        metadatas = []
+
         for passage_text, passage_metadata in connector.generate_passages(file_metadata, chunk_size=embedding_config.embedding_chunk_size):
             # for some reason, llama index parsers sometimes return empty strings
             if len(passage_text) == 0:
@@ -104,24 +71,74 @@ async def load_data(connector: DataConnector, source: Source, passage_manager: P
                 )
                 continue
 
-            # get embedding
             texts.append(passage_text)
+            metadatas.append(passage_metadata)
+
             if len(texts) >= EMBEDDING_BATCH_SIZE:
-                passages = await generate_embeddings(texts, embedding_config)
+                # Process the batch
+                embeddings = await client.request_embeddings(texts, embedding_config)
+                passages = []
+
+                for text, embedding, passage_metadata in zip(texts, embeddings, metadatas):
+                    passage = Passage(
+                        text=text,
+                        file_id=file_metadata.id,
+                        source_id=source.id,
+                        metadata=passage_metadata,
+                        organization_id=source.organization_id,
+                        embedding_config=source.embedding_config,
+                        embedding=embedding,
+                    )
+                    hashable_embedding = tuple(passage.embedding)
+                    file_name = file_metadata.file_name
+                    if hashable_embedding in embedding_to_document_name:
+                        typer.secho(
+                            f"Warning: Duplicate embedding found for passage in {file_name} (already exists in {embedding_to_document_name[hashable_embedding]}), skipping insert into VectorDB.",
+                            fg=typer.colors.YELLOW,
+                        )
+                        continue
+
+                    passages.append(passage)
+                    embedding_to_document_name[hashable_embedding] = file_name
+
+                # insert passages into passage store
+                await passage_manager.create_many_passages_async(passages, actor)
+                passage_count += len(passages)
+
+                # Reset for next batch
                 texts = []
-            else:
-                continue
+                metadatas = []
+
+        # Process final remaining texts for this file
+        if len(texts) > 0:
+            embeddings = await client.request_embeddings(texts, embedding_config)
+            passages = []
+
+            for text, embedding, passage_metadata in zip(texts, embeddings, metadatas):
+                passage = Passage(
+                    text=text,
+                    file_id=file_metadata.id,
+                    source_id=source.id,
+                    metadata=passage_metadata,
+                    organization_id=source.organization_id,
+                    embedding_config=source.embedding_config,
+                    embedding=embedding,
+                )
+                hashable_embedding = tuple(passage.embedding)
+                file_name = file_metadata.file_name
+                if hashable_embedding in embedding_to_document_name:
+                    typer.secho(
+                        f"Warning: Duplicate embedding found for passage in {file_name} (already exists in {embedding_to_document_name[hashable_embedding]}), skipping insert into VectorDB.",
+                        fg=typer.colors.YELLOW,
+                    )
+                    continue
+
+                passages.append(passage)
+                embedding_to_document_name[hashable_embedding] = file_name
 
-            # insert passages into passage store
             await passage_manager.create_many_passages_async(passages, actor)
             passage_count += len(passages)
 
-    # final remaining
-    if len(texts) > 0:
-        passages = await generate_embeddings(texts, embedding_config)
-        await passage_manager.create_many_passages_async(passages, actor)
-        passage_count += len(passages)
-
     return passage_count, file_count
 
 

letta_nightly-0.11.4.dev20250820213507/letta/embeddings.py

@@ -0,0 +1,53 @@
+from typing import List
+
+import tiktoken
+
+from letta.constants import EMBEDDING_TO_TOKENIZER_DEFAULT, EMBEDDING_TO_TOKENIZER_MAP
+from letta.utils import printd
+
+
+def parse_and_chunk_text(text: str, chunk_size: int) -> List[str]:
+    from llama_index.core import Document as LlamaIndexDocument
+    from llama_index.core.node_parser import SentenceSplitter
+
+    parser = SentenceSplitter(chunk_size=chunk_size)
+    llama_index_docs = [LlamaIndexDocument(text=text)]
+    nodes = parser.get_nodes_from_documents(llama_index_docs)
+    return [n.text for n in nodes]
+
+
+def truncate_text(text: str, max_length: int, encoding) -> str:
+    # truncate the text based on max_length and encoding
+    encoded_text = encoding.encode(text)[:max_length]
+    return encoding.decode(encoded_text)
+
+
+def check_and_split_text(text: str, embedding_model: str) -> List[str]:
+    """Split text into chunks of max_length tokens or less"""
+
+    if embedding_model in EMBEDDING_TO_TOKENIZER_MAP:
+        encoding = tiktoken.get_encoding(EMBEDDING_TO_TOKENIZER_MAP[embedding_model])
+    else:
+        print(f"Warning: couldn't find tokenizer for model {embedding_model}, using default tokenizer {EMBEDDING_TO_TOKENIZER_DEFAULT}")
+        encoding = tiktoken.get_encoding(EMBEDDING_TO_TOKENIZER_DEFAULT)
+
+    num_tokens = len(encoding.encode(text))
+
+    # determine max length
+    if hasattr(encoding, "max_length"):
+        # TODO(fix) this is broken
+        max_length = encoding.max_length
+    else:
+        # TODO: figure out the real number
+        printd(f"Warning: couldn't find max_length for tokenizer {embedding_model}, using default max_length 8191")
+        max_length = 8191
+
+    # truncate text if too long
+    if num_tokens > max_length:
+        print(f"Warning: text is too long ({num_tokens} tokens), truncating to {max_length} tokens.")
+        # First, apply any necessary formatting
+        formatted_text = format_text(text, embedding_model)
+        # Then truncate
+        text = truncate_text(formatted_text, max_length, encoding)
+
+    return [text]

{letta_nightly-0.11.3.dev20250819104229 → letta_nightly-0.11.4.dev20250820213507}/letta/errors.py

@@ -236,5 +236,33 @@ class AgentFileExportError(Exception):
     """Exception raised during agent file export operations"""
 
 
+class AgentNotFoundForExportError(AgentFileExportError):
+    """Exception raised when requested agents are not found during export"""
+
+    def __init__(self, missing_ids: List[str]):
+        self.missing_ids = missing_ids
+        super().__init__(f"The following agent IDs were not found: {missing_ids}")
+
+
+class AgentExportIdMappingError(AgentFileExportError):
+    """Exception raised when ID mapping fails during export conversion"""
+
+    def __init__(self, db_id: str, entity_type: str):
+        self.db_id = db_id
+        self.entity_type = entity_type
+        super().__init__(
+            f"Unexpected new {entity_type} ID '{db_id}' encountered during conversion. "
+            f"All IDs should have been mapped during agent processing."
+        )
+
+
+class AgentExportProcessingError(AgentFileExportError):
+    """Exception raised when general export processing fails"""
+
+    def __init__(self, message: str, original_error: Optional[Exception] = None):
+        self.original_error = original_error
+        super().__init__(f"Export failed: {message}")
+
+
 class AgentFileImportError(Exception):
     """Exception raised during agent file import operations"""