letta-nightly 0.8.13.dev20250714104447__py3-none-any.whl → 0.8.15.dev20250715080149__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of letta-nightly might be problematic. Click here for more details.

Files changed (36) hide show
  1. letta/__init__.py +1 -1
  2. letta/constants.py +6 -0
  3. letta/functions/function_sets/base.py +2 -2
  4. letta/functions/function_sets/files.py +11 -11
  5. letta/helpers/decorators.py +1 -1
  6. letta/helpers/pinecone_utils.py +164 -11
  7. letta/orm/agent.py +1 -1
  8. letta/orm/file.py +2 -17
  9. letta/orm/files_agents.py +9 -10
  10. letta/orm/organization.py +0 -4
  11. letta/orm/passage.py +0 -10
  12. letta/orm/source.py +3 -20
  13. letta/prompts/system/memgpt_v2_chat.txt +28 -10
  14. letta/schemas/file.py +1 -0
  15. letta/schemas/memory.py +2 -2
  16. letta/server/rest_api/routers/v1/agents.py +4 -4
  17. letta/server/rest_api/routers/v1/messages.py +2 -6
  18. letta/server/rest_api/routers/v1/sources.py +3 -3
  19. letta/server/server.py +0 -3
  20. letta/services/agent_manager.py +194 -147
  21. letta/services/block_manager.py +18 -18
  22. letta/services/context_window_calculator/context_window_calculator.py +15 -10
  23. letta/services/context_window_calculator/token_counter.py +40 -0
  24. letta/services/file_manager.py +37 -0
  25. letta/services/file_processor/chunker/line_chunker.py +17 -0
  26. letta/services/file_processor/embedder/openai_embedder.py +50 -5
  27. letta/services/files_agents_manager.py +12 -2
  28. letta/services/group_manager.py +11 -11
  29. letta/services/source_manager.py +19 -3
  30. letta/services/tool_executor/core_tool_executor.py +2 -2
  31. letta/services/tool_executor/files_tool_executor.py +6 -1
  32. {letta_nightly-0.8.13.dev20250714104447.dist-info → letta_nightly-0.8.15.dev20250715080149.dist-info}/METADATA +1 -1
  33. {letta_nightly-0.8.13.dev20250714104447.dist-info → letta_nightly-0.8.15.dev20250715080149.dist-info}/RECORD +36 -36
  34. {letta_nightly-0.8.13.dev20250714104447.dist-info → letta_nightly-0.8.15.dev20250715080149.dist-info}/LICENSE +0 -0
  35. {letta_nightly-0.8.13.dev20250714104447.dist-info → letta_nightly-0.8.15.dev20250715080149.dist-info}/WHEEL +0 -0
  36. {letta_nightly-0.8.13.dev20250714104447.dist-info → letta_nightly-0.8.15.dev20250715080149.dist-info}/entry_points.txt +0 -0
letta/__init__.py CHANGED
@@ -5,7 +5,7 @@ try:
5
5
  __version__ = version("letta")
6
6
  except PackageNotFoundError:
7
7
  # Fallback for development installations
8
- __version__ = "0.8.13"
8
+ __version__ = "0.8.15"
9
9
 
10
10
  if os.environ.get("LETTA_VERSION"):
11
11
  __version__ = os.environ["LETTA_VERSION"]
letta/constants.py CHANGED
@@ -372,3 +372,9 @@ PINECONE_METRIC = "cosine"
372
372
  PINECONE_CLOUD = "aws"
373
373
  PINECONE_REGION = "us-east-1"
374
374
  PINECONE_MAX_BATCH_SIZE = 96
375
+
376
+ # retry configuration
377
+ PINECONE_MAX_RETRY_ATTEMPTS = 3
378
+ PINECONE_RETRY_BASE_DELAY = 1.0 # seconds
379
+ PINECONE_RETRY_MAX_DELAY = 60.0 # seconds
380
+ PINECONE_RETRY_BACKOFF_FACTOR = 2.0
@@ -135,7 +135,7 @@ def core_memory_append(agent_state: "AgentState", label: str, content: str) -> O
135
135
  Append to the contents of core memory.
136
136
 
137
137
  Args:
138
- label (str): Section of the memory to be edited (persona or human).
138
+ label (str): Section of the memory to be edited.
139
139
  content (str): Content to write to the memory. All unicode (including emojis) are supported.
140
140
 
141
141
  Returns:
@@ -152,7 +152,7 @@ def core_memory_replace(agent_state: "AgentState", label: str, old_content: str,
152
152
  Replace the contents of core memory. To delete memories, use an empty string for new_content.
153
153
 
154
154
  Args:
155
- label (str): Section of the memory to be edited (persona or human).
155
+ label (str): Section of the memory to be edited.
156
156
  old_content (str): String to replace. Must be an exact match.
157
157
  new_content (str): Content to write to the memory. All unicode (including emojis) are supported.
158
158
 
@@ -46,12 +46,12 @@ async def grep_files(
46
46
  context_lines: Optional[int] = 3,
47
47
  ) -> str:
48
48
  """
49
- Grep tool to search files across data sources using a keyword or regex pattern.
49
+ Searches file contents for pattern matches with surrounding context.
50
50
 
51
- Use this when you want to:
52
- - Quickly find occurrences of a variable, function, or keyword
53
- - Locate log messages, error codes, or TODOs across files
54
- - Understand surrounding code by including `context_lines`
51
+ Ideal for:
52
+ - Finding specific code elements (variables, functions, keywords)
53
+ - Locating error messages or specific text across multiple files
54
+ - Examining code in context to understand usage patterns
55
55
 
56
56
  Args:
57
57
  pattern (str): Keyword or regex pattern to search within file contents.
@@ -67,15 +67,15 @@ async def grep_files(
67
67
 
68
68
  async def semantic_search_files(agent_state: "AgentState", query: str, limit: int = 5) -> List["FileMetadata"]:
69
69
  """
70
- Get list of most relevant chunks from any file using vector/embedding search.
70
+ Searches file contents using semantic meaning rather than exact matches.
71
71
 
72
- Use this when you want to:
73
- - Find related content that without using exact keywords (e.g., conceptually similar sections)
74
- - Look up high-level descriptions, documentation, or config patterns
75
- - Perform fuzzy search when grep isn't sufficient
72
+ Ideal for:
73
+ - Finding conceptually related information across files
74
+ - Discovering relevant content without knowing exact keywords
75
+ - Locating files with similar topics or themes
76
76
 
77
77
  Args:
78
- query (str): The search query.
78
+ query (str): The search query text to find semantically similar content.
79
79
  limit: Maximum number of results to return (default: 5)
80
80
 
81
81
  Returns:
@@ -152,7 +152,7 @@ def async_redis_cache(
152
152
  def get_cache_key(*args, **kwargs):
153
153
  return f"{prefix}:{key_func(*args, **kwargs)}"
154
154
 
155
- # async_wrapper.cache_invalidate = invalidate
155
+ async_wrapper.cache_invalidate = invalidate
156
156
  async_wrapper.cache_key_func = get_cache_key
157
157
  async_wrapper.cache_stats = stats
158
158
  return async_wrapper
@@ -1,8 +1,20 @@
1
+ import asyncio
2
+ import random
3
+ import time
4
+ from functools import wraps
1
5
  from typing import Any, Dict, List
2
6
 
7
+ from letta.otel.tracing import trace_method
8
+
3
9
  try:
4
10
  from pinecone import IndexEmbed, PineconeAsyncio
5
- from pinecone.exceptions.exceptions import NotFoundException
11
+ from pinecone.exceptions.exceptions import (
12
+ ForbiddenException,
13
+ NotFoundException,
14
+ PineconeApiException,
15
+ ServiceException,
16
+ UnauthorizedException,
17
+ )
6
18
 
7
19
  PINECONE_AVAILABLE = True
8
20
  except ImportError:
@@ -12,8 +24,12 @@ from letta.constants import (
12
24
  PINECONE_CLOUD,
13
25
  PINECONE_EMBEDDING_MODEL,
14
26
  PINECONE_MAX_BATCH_SIZE,
27
+ PINECONE_MAX_RETRY_ATTEMPTS,
15
28
  PINECONE_METRIC,
16
29
  PINECONE_REGION,
30
+ PINECONE_RETRY_BACKOFF_FACTOR,
31
+ PINECONE_RETRY_BASE_DELAY,
32
+ PINECONE_RETRY_MAX_DELAY,
17
33
  PINECONE_TEXT_FIELD_NAME,
18
34
  )
19
35
  from letta.log import get_logger
@@ -23,6 +39,87 @@ from letta.settings import settings
23
39
  logger = get_logger(__name__)
24
40
 
25
41
 
42
+ def pinecone_retry(
43
+ max_attempts: int = PINECONE_MAX_RETRY_ATTEMPTS,
44
+ base_delay: float = PINECONE_RETRY_BASE_DELAY,
45
+ max_delay: float = PINECONE_RETRY_MAX_DELAY,
46
+ backoff_factor: float = PINECONE_RETRY_BACKOFF_FACTOR,
47
+ ):
48
+ """
49
+ Decorator to retry Pinecone operations with exponential backoff.
50
+
51
+ Args:
52
+ max_attempts: Maximum number of retry attempts
53
+ base_delay: Base delay in seconds for the first retry
54
+ max_delay: Maximum delay in seconds between retries
55
+ backoff_factor: Factor to increase delay after each failed attempt
56
+ """
57
+
58
+ def decorator(func):
59
+ @wraps(func)
60
+ async def wrapper(*args, **kwargs):
61
+ operation_name = func.__name__
62
+ start_time = time.time()
63
+
64
+ for attempt in range(max_attempts):
65
+ try:
66
+ logger.debug(f"[Pinecone] Starting {operation_name} (attempt {attempt + 1}/{max_attempts})")
67
+ result = await func(*args, **kwargs)
68
+
69
+ execution_time = time.time() - start_time
70
+ logger.info(f"[Pinecone] {operation_name} completed successfully in {execution_time:.2f}s")
71
+ return result
72
+
73
+ except (ServiceException, PineconeApiException) as e:
74
+ # retryable server errors
75
+ if attempt == max_attempts - 1:
76
+ execution_time = time.time() - start_time
77
+ logger.error(f"[Pinecone] {operation_name} failed after {max_attempts} attempts in {execution_time:.2f}s: {str(e)}")
78
+ raise
79
+
80
+ # calculate delay with exponential backoff and jitter
81
+ delay = min(base_delay * (backoff_factor**attempt), max_delay)
82
+ jitter = random.uniform(0, delay * 0.1) # add up to 10% jitter
83
+ total_delay = delay + jitter
84
+
85
+ logger.warning(
86
+ f"[Pinecone] {operation_name} failed (attempt {attempt + 1}/{max_attempts}): {str(e)}. Retrying in {total_delay:.2f}s"
87
+ )
88
+ await asyncio.sleep(total_delay)
89
+
90
+ except (UnauthorizedException, ForbiddenException) as e:
91
+ # non-retryable auth errors
92
+ execution_time = time.time() - start_time
93
+ logger.error(f"[Pinecone] {operation_name} failed with auth error in {execution_time:.2f}s: {str(e)}")
94
+ raise
95
+
96
+ except NotFoundException as e:
97
+ # non-retryable not found errors
98
+ execution_time = time.time() - start_time
99
+ logger.warning(f"[Pinecone] {operation_name} failed with not found error in {execution_time:.2f}s: {str(e)}")
100
+ raise
101
+
102
+ except Exception as e:
103
+ # other unexpected errors - retry once then fail
104
+ if attempt == max_attempts - 1:
105
+ execution_time = time.time() - start_time
106
+ logger.error(f"[Pinecone] {operation_name} failed after {max_attempts} attempts in {execution_time:.2f}s: {str(e)}")
107
+ raise
108
+
109
+ delay = min(base_delay * (backoff_factor**attempt), max_delay)
110
+ jitter = random.uniform(0, delay * 0.1)
111
+ total_delay = delay + jitter
112
+
113
+ logger.warning(
114
+ f"[Pinecone] {operation_name} failed with unexpected error (attempt {attempt + 1}/{max_attempts}): {str(e)}. Retrying in {total_delay:.2f}s"
115
+ )
116
+ await asyncio.sleep(total_delay)
117
+
118
+ return wrapper
119
+
120
+ return decorator
121
+
122
+
26
123
  def should_use_pinecone(verbose: bool = False):
27
124
  if verbose:
28
125
  logger.info(
@@ -44,29 +141,42 @@ def should_use_pinecone(verbose: bool = False):
44
141
  )
45
142
 
46
143
 
144
+ @pinecone_retry()
145
+ @trace_method
47
146
  async def upsert_pinecone_indices():
48
147
  if not PINECONE_AVAILABLE:
49
148
  raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
50
149
 
51
- for index_name in get_pinecone_indices():
150
+ indices = get_pinecone_indices()
151
+ logger.info(f"[Pinecone] Upserting {len(indices)} indices: {indices}")
152
+
153
+ for index_name in indices:
52
154
  async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
53
155
  if not await pc.has_index(index_name):
156
+ logger.info(f"[Pinecone] Creating index {index_name} with model {PINECONE_EMBEDDING_MODEL}")
54
157
  await pc.create_index_for_model(
55
158
  name=index_name,
56
159
  cloud=PINECONE_CLOUD,
57
160
  region=PINECONE_REGION,
58
161
  embed=IndexEmbed(model=PINECONE_EMBEDDING_MODEL, field_map={"text": PINECONE_TEXT_FIELD_NAME}, metric=PINECONE_METRIC),
59
162
  )
163
+ logger.info(f"[Pinecone] Successfully created index {index_name}")
164
+ else:
165
+ logger.debug(f"[Pinecone] Index {index_name} already exists")
60
166
 
61
167
 
62
168
  def get_pinecone_indices() -> List[str]:
63
169
  return [settings.pinecone_agent_index, settings.pinecone_source_index]
64
170
 
65
171
 
172
+ @pinecone_retry()
173
+ @trace_method
66
174
  async def upsert_file_records_to_pinecone_index(file_id: str, source_id: str, chunks: List[str], actor: User):
67
175
  if not PINECONE_AVAILABLE:
68
176
  raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
69
177
 
178
+ logger.info(f"[Pinecone] Preparing to upsert {len(chunks)} chunks for file {file_id} source {source_id}")
179
+
70
180
  records = []
71
181
  for i, chunk in enumerate(chunks):
72
182
  record = {
@@ -77,14 +187,19 @@ async def upsert_file_records_to_pinecone_index(file_id: str, source_id: str, ch
77
187
  }
78
188
  records.append(record)
79
189
 
190
+ logger.debug(f"[Pinecone] Created {len(records)} records for file {file_id}")
80
191
  return await upsert_records_to_pinecone_index(records, actor)
81
192
 
82
193
 
194
+ @pinecone_retry()
195
+ @trace_method
83
196
  async def delete_file_records_from_pinecone_index(file_id: str, actor: User):
84
197
  if not PINECONE_AVAILABLE:
85
198
  raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
86
199
 
87
200
  namespace = actor.organization_id
201
+ logger.info(f"[Pinecone] Deleting records for file {file_id} from index {settings.pinecone_source_index} namespace {namespace}")
202
+
88
203
  try:
89
204
  async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
90
205
  description = await pc.describe_index(name=settings.pinecone_source_index)
@@ -95,48 +210,72 @@ async def delete_file_records_from_pinecone_index(file_id: str, actor: User):
95
210
  },
96
211
  namespace=namespace,
97
212
  )
213
+ logger.info(f"[Pinecone] Successfully deleted records for file {file_id}")
98
214
  except NotFoundException:
99
- logger.warning(f"Pinecone namespace {namespace} not found for {file_id} and {actor.organization_id}")
215
+ logger.warning(f"[Pinecone] Namespace {namespace} not found for file {file_id} and org {actor.organization_id}")
100
216
 
101
217
 
218
+ @pinecone_retry()
219
+ @trace_method
102
220
  async def delete_source_records_from_pinecone_index(source_id: str, actor: User):
103
221
  if not PINECONE_AVAILABLE:
104
222
  raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
105
223
 
106
224
  namespace = actor.organization_id
225
+ logger.info(f"[Pinecone] Deleting records for source {source_id} from index {settings.pinecone_source_index} namespace {namespace}")
226
+
107
227
  try:
108
228
  async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
109
229
  description = await pc.describe_index(name=settings.pinecone_source_index)
110
230
  async with pc.IndexAsyncio(host=description.index.host) as dense_index:
111
231
  await dense_index.delete(filter={"source_id": {"$eq": source_id}}, namespace=namespace)
232
+ logger.info(f"[Pinecone] Successfully deleted records for source {source_id}")
112
233
  except NotFoundException:
113
- logger.warning(f"Pinecone namespace {namespace} not found for {source_id} and {actor.organization_id}")
234
+ logger.warning(f"[Pinecone] Namespace {namespace} not found for source {source_id} and org {actor.organization_id}")
114
235
 
115
236
 
237
+ @pinecone_retry()
238
+ @trace_method
116
239
  async def upsert_records_to_pinecone_index(records: List[dict], actor: User):
117
240
  if not PINECONE_AVAILABLE:
118
241
  raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
119
242
 
243
+ logger.info(f"[Pinecone] Upserting {len(records)} records to index {settings.pinecone_source_index} for org {actor.organization_id}")
244
+
120
245
  async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
121
246
  description = await pc.describe_index(name=settings.pinecone_source_index)
122
247
  async with pc.IndexAsyncio(host=description.index.host) as dense_index:
123
- # Process records in batches to avoid exceeding Pinecone limits
248
+ # process records in batches to avoid exceeding pinecone limits
249
+ total_batches = (len(records) + PINECONE_MAX_BATCH_SIZE - 1) // PINECONE_MAX_BATCH_SIZE
250
+ logger.debug(f"[Pinecone] Processing {total_batches} batches of max {PINECONE_MAX_BATCH_SIZE} records each")
251
+
124
252
  for i in range(0, len(records), PINECONE_MAX_BATCH_SIZE):
125
253
  batch = records[i : i + PINECONE_MAX_BATCH_SIZE]
254
+ batch_num = (i // PINECONE_MAX_BATCH_SIZE) + 1
255
+
256
+ logger.debug(f"[Pinecone] Upserting batch {batch_num}/{total_batches} with {len(batch)} records")
126
257
  await dense_index.upsert_records(actor.organization_id, batch)
127
258
 
259
+ logger.info(f"[Pinecone] Successfully upserted all {len(records)} records in {total_batches} batches")
260
+
128
261
 
262
+ @pinecone_retry()
263
+ @trace_method
129
264
  async def search_pinecone_index(query: str, limit: int, filter: Dict[str, Any], actor: User) -> Dict[str, Any]:
130
265
  if not PINECONE_AVAILABLE:
131
266
  raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
132
267
 
268
+ namespace = actor.organization_id
269
+ logger.info(
270
+ f"[Pinecone] Searching index {settings.pinecone_source_index} namespace {namespace} with query length {len(query)} chars, limit {limit}"
271
+ )
272
+ logger.debug(f"[Pinecone] Search filter: {filter}")
273
+
133
274
  async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
134
275
  description = await pc.describe_index(name=settings.pinecone_source_index)
135
276
  async with pc.IndexAsyncio(host=description.index.host) as dense_index:
136
- namespace = actor.organization_id
137
-
138
277
  try:
139
- # Search the dense index with reranking
278
+ # search the dense index with reranking
140
279
  search_results = await dense_index.search(
141
280
  namespace=namespace,
142
281
  query={
@@ -146,17 +285,26 @@ async def search_pinecone_index(query: str, limit: int, filter: Dict[str, Any],
146
285
  },
147
286
  rerank={"model": "bge-reranker-v2-m3", "top_n": limit, "rank_fields": [PINECONE_TEXT_FIELD_NAME]},
148
287
  )
288
+
289
+ result_count = len(search_results.get("matches", []))
290
+ logger.info(f"[Pinecone] Search completed, found {result_count} matches")
149
291
  return search_results
292
+
150
293
  except Exception as e:
151
- logger.warning(f"Failed to search Pinecone namespace {actor.organization_id}: {str(e)}")
294
+ logger.warning(f"[Pinecone] Failed to search namespace {namespace}: {str(e)}")
152
295
  raise e
153
296
 
154
297
 
298
+ @pinecone_retry()
299
+ @trace_method
155
300
  async def list_pinecone_index_for_files(file_id: str, actor: User, limit: int = None, pagination_token: str = None) -> List[str]:
156
301
  if not PINECONE_AVAILABLE:
157
302
  raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
158
303
 
159
304
  namespace = actor.organization_id
305
+ logger.info(f"[Pinecone] Listing records for file {file_id} from index {settings.pinecone_source_index} namespace {namespace}")
306
+ logger.debug(f"[Pinecone] List params - limit: {limit}, pagination_token: {pagination_token}")
307
+
160
308
  try:
161
309
  async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
162
310
  description = await pc.describe_index(name=settings.pinecone_source_index)
@@ -172,9 +320,14 @@ async def list_pinecone_index_for_files(file_id: str, actor: User, limit: int =
172
320
  result = []
173
321
  async for ids in dense_index.list(**kwargs):
174
322
  result.extend(ids)
323
+
324
+ logger.info(f"[Pinecone] Successfully listed {len(result)} records for file {file_id}")
175
325
  return result
326
+
176
327
  except Exception as e:
177
- logger.warning(f"Failed to list Pinecone namespace {actor.organization_id}: {str(e)}")
328
+ logger.warning(f"[Pinecone] Failed to list records for file {file_id} in namespace {namespace}: {str(e)}")
178
329
  raise e
330
+
179
331
  except NotFoundException:
180
- logger.warning(f"Pinecone namespace {namespace} not found for {file_id} and {actor.organization_id}")
332
+ logger.warning(f"[Pinecone] Namespace {namespace} not found for file {file_id} and org {actor.organization_id}")
333
+ return []
letta/orm/agent.py CHANGED
@@ -314,7 +314,7 @@ class Agent(SqlalchemyBase, OrganizationMixin, AsyncAttrs):
314
314
  state["sources"] = [s.to_pydantic() for s in sources]
315
315
  state["memory"] = Memory(
316
316
  blocks=[m.to_pydantic() for m in memory],
317
- file_blocks=[block for b in self.file_agents if (block := b.to_pydantic_block()) is not None],
317
+ file_blocks=[block for b in file_agents if (block := b.to_pydantic_block()) is not None],
318
318
  prompt_template=get_prompt_template_for_agent_type(self.agent_type),
319
319
  )
320
320
  state["identity_ids"] = [i.id for i in identities]
letta/orm/file.py CHANGED
@@ -1,5 +1,5 @@
1
1
  import uuid
2
- from typing import TYPE_CHECKING, List, Optional
2
+ from typing import TYPE_CHECKING, Optional
3
3
 
4
4
  from sqlalchemy import ForeignKey, Index, Integer, String, Text, UniqueConstraint, desc
5
5
  from sqlalchemy.ext.asyncio import AsyncAttrs
@@ -11,10 +11,7 @@ from letta.schemas.enums import FileProcessingStatus
11
11
  from letta.schemas.file import FileMetadata as PydanticFileMetadata
12
12
 
13
13
  if TYPE_CHECKING:
14
- from letta.orm.files_agents import FileAgent
15
- from letta.orm.organization import Organization
16
- from letta.orm.passage import SourcePassage
17
- from letta.orm.source import Source
14
+ pass
18
15
 
19
16
 
20
17
  # TODO: Note that this is NOT organization scoped, this is potentially dangerous if we misuse this
@@ -64,18 +61,6 @@ class FileMetadata(SqlalchemyBase, OrganizationMixin, SourceMixin, AsyncAttrs):
64
61
  chunks_embedded: Mapped[Optional[int]] = mapped_column(Integer, nullable=True, doc="Number of chunks that have been embedded.")
65
62
 
66
63
  # relationships
67
- organization: Mapped["Organization"] = relationship("Organization", back_populates="files", lazy="selectin")
68
- source: Mapped["Source"] = relationship("Source", back_populates="files", lazy="selectin")
69
- source_passages: Mapped[List["SourcePassage"]] = relationship(
70
- "SourcePassage", back_populates="file", lazy="selectin", cascade="all, delete-orphan"
71
- )
72
- file_agents: Mapped[List["FileAgent"]] = relationship(
73
- "FileAgent",
74
- back_populates="file",
75
- lazy="selectin",
76
- cascade="all, delete-orphan",
77
- passive_deletes=True, # ← add this
78
- )
79
64
  content: Mapped[Optional["FileContent"]] = relationship(
80
65
  "FileContent",
81
66
  uselist=False,
letta/orm/files_agents.py CHANGED
@@ -12,7 +12,7 @@ from letta.schemas.block import Block as PydanticBlock
12
12
  from letta.schemas.file import FileAgent as PydanticFileAgent
13
13
 
14
14
  if TYPE_CHECKING:
15
- from letta.orm.file import FileMetadata
15
+ pass
16
16
 
17
17
 
18
18
  class FileAgent(SqlalchemyBase, OrganizationMixin):
@@ -55,6 +55,12 @@ class FileAgent(SqlalchemyBase, OrganizationMixin):
55
55
  nullable=False,
56
56
  doc="ID of the agent",
57
57
  )
58
+ source_id: Mapped[str] = mapped_column(
59
+ String,
60
+ ForeignKey("sources.id", ondelete="CASCADE"),
61
+ nullable=False,
62
+ doc="ID of the source (denormalized from files.source_id)",
63
+ )
58
64
 
59
65
  file_name: Mapped[str] = mapped_column(
60
66
  String,
@@ -78,13 +84,6 @@ class FileAgent(SqlalchemyBase, OrganizationMixin):
78
84
  back_populates="file_agents",
79
85
  lazy="selectin",
80
86
  )
81
- file: Mapped["FileMetadata"] = relationship(
82
- "FileMetadata",
83
- foreign_keys=[file_id],
84
- lazy="selectin",
85
- back_populates="file_agents",
86
- passive_deletes=True, # ← add this
87
- )
88
87
 
89
88
  # TODO: This is temporary as we figure out if we want FileBlock as a first class citizen
90
89
  def to_pydantic_block(self) -> PydanticBlock:
@@ -99,8 +98,8 @@ class FileAgent(SqlalchemyBase, OrganizationMixin):
99
98
  return PydanticBlock(
100
99
  organization_id=self.organization_id,
101
100
  value=visible_content,
102
- label=self.file.file_name,
101
+ label=self.file_name, # use denormalized file_name instead of self.file.file_name
103
102
  read_only=True,
104
- metadata={"source_id": self.file.source_id},
103
+ metadata={"source_id": self.source_id}, # use denormalized source_id
105
104
  limit=CORE_MEMORY_SOURCE_CHAR_LIMIT,
106
105
  )
letta/orm/organization.py CHANGED
@@ -9,7 +9,6 @@ if TYPE_CHECKING:
9
9
  from letta.orm.agent import Agent
10
10
  from letta.orm.agent_passage import AgentPassage
11
11
  from letta.orm.block import Block
12
- from letta.orm.file import FileMetadata
13
12
  from letta.orm.group import Group
14
13
  from letta.orm.identity import Identity
15
14
  from letta.orm.llm_batch_item import LLMBatchItem
@@ -18,7 +17,6 @@ if TYPE_CHECKING:
18
17
  from letta.orm.provider import Provider
19
18
  from letta.orm.sandbox_config import AgentEnvironmentVariable, SandboxConfig
20
19
  from letta.orm.sandbox_environment_variable import SandboxEnvironmentVariable
21
- from letta.orm.source import Source
22
20
  from letta.orm.source_passage import SourcePassage
23
21
  from letta.orm.tool import Tool
24
22
  from letta.orm.user import User
@@ -38,8 +36,6 @@ class Organization(SqlalchemyBase):
38
36
  tools: Mapped[List["Tool"]] = relationship("Tool", back_populates="organization", cascade="all, delete-orphan")
39
37
  # mcp_servers: Mapped[List["MCPServer"]] = relationship("MCPServer", back_populates="organization", cascade="all, delete-orphan")
40
38
  blocks: Mapped[List["Block"]] = relationship("Block", back_populates="organization", cascade="all, delete-orphan")
41
- sources: Mapped[List["Source"]] = relationship("Source", back_populates="organization", cascade="all, delete-orphan")
42
- files: Mapped[List["FileMetadata"]] = relationship("FileMetadata", back_populates="organization", cascade="all, delete-orphan")
43
39
  sandbox_configs: Mapped[List["SandboxConfig"]] = relationship(
44
40
  "SandboxConfig", back_populates="organization", cascade="all, delete-orphan"
45
41
  )
letta/orm/passage.py CHANGED
@@ -49,11 +49,6 @@ class SourcePassage(BasePassage, FileMixin, SourceMixin):
49
49
 
50
50
  file_name: Mapped[str] = mapped_column(doc="The name of the file that this passage was derived from")
51
51
 
52
- @declared_attr
53
- def file(cls) -> Mapped["FileMetadata"]:
54
- """Relationship to file"""
55
- return relationship("FileMetadata", back_populates="source_passages", lazy="selectin")
56
-
57
52
  @declared_attr
58
53
  def organization(cls) -> Mapped["Organization"]:
59
54
  return relationship("Organization", back_populates="source_passages", lazy="selectin")
@@ -74,11 +69,6 @@ class SourcePassage(BasePassage, FileMixin, SourceMixin):
74
69
  {"extend_existing": True},
75
70
  )
76
71
 
77
- @declared_attr
78
- def source(cls) -> Mapped["Source"]:
79
- """Relationship to source"""
80
- return relationship("Source", back_populates="passages", lazy="selectin", passive_deletes=True)
81
-
82
72
 
83
73
  class AgentPassage(BasePassage, AgentMixin):
84
74
  """Passages created by agents as archival memories"""
letta/orm/source.py CHANGED
@@ -1,9 +1,8 @@
1
- from typing import TYPE_CHECKING, List, Optional
1
+ from typing import TYPE_CHECKING, Optional
2
2
 
3
3
  from sqlalchemy import JSON, Index, UniqueConstraint
4
- from sqlalchemy.orm import Mapped, mapped_column, relationship
4
+ from sqlalchemy.orm import Mapped, mapped_column
5
5
 
6
- from letta.orm import FileMetadata
7
6
  from letta.orm.custom_columns import EmbeddingConfigColumn
8
7
  from letta.orm.mixins import OrganizationMixin
9
8
  from letta.orm.sqlalchemy_base import SqlalchemyBase
@@ -11,10 +10,7 @@ from letta.schemas.embedding_config import EmbeddingConfig
11
10
  from letta.schemas.source import Source as PydanticSource
12
11
 
13
12
  if TYPE_CHECKING:
14
- from letta.orm.agent import Agent
15
- from letta.orm.file import FileMetadata
16
- from letta.orm.organization import Organization
17
- from letta.orm.passage import SourcePassage
13
+ pass
18
14
 
19
15
 
20
16
  class Source(SqlalchemyBase, OrganizationMixin):
@@ -34,16 +30,3 @@ class Source(SqlalchemyBase, OrganizationMixin):
34
30
  instructions: Mapped[str] = mapped_column(nullable=True, doc="instructions for how to use the source")
35
31
  embedding_config: Mapped[EmbeddingConfig] = mapped_column(EmbeddingConfigColumn, doc="Configuration settings for embedding.")
36
32
  metadata_: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True, doc="metadata for the source.")
37
-
38
- # relationships
39
- organization: Mapped["Organization"] = relationship("Organization", back_populates="sources")
40
- files: Mapped[List["FileMetadata"]] = relationship("FileMetadata", back_populates="source", cascade="all, delete-orphan")
41
- passages: Mapped[List["SourcePassage"]] = relationship("SourcePassage", back_populates="source", cascade="all, delete-orphan")
42
- agents: Mapped[List["Agent"]] = relationship(
43
- "Agent",
44
- secondary="sources_agents",
45
- back_populates="sources",
46
- lazy="selectin",
47
- cascade="save-update", # Only propagate save and update operations
48
- passive_deletes=True, # Let the database handle deletions
49
- )