letta-nightly 0.8.13.dev20250714104447__py3-none-any.whl → 0.8.15.dev20250715080149__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of letta-nightly might be problematic. Click here for more details.
- letta/__init__.py +1 -1
- letta/constants.py +6 -0
- letta/functions/function_sets/base.py +2 -2
- letta/functions/function_sets/files.py +11 -11
- letta/helpers/decorators.py +1 -1
- letta/helpers/pinecone_utils.py +164 -11
- letta/orm/agent.py +1 -1
- letta/orm/file.py +2 -17
- letta/orm/files_agents.py +9 -10
- letta/orm/organization.py +0 -4
- letta/orm/passage.py +0 -10
- letta/orm/source.py +3 -20
- letta/prompts/system/memgpt_v2_chat.txt +28 -10
- letta/schemas/file.py +1 -0
- letta/schemas/memory.py +2 -2
- letta/server/rest_api/routers/v1/agents.py +4 -4
- letta/server/rest_api/routers/v1/messages.py +2 -6
- letta/server/rest_api/routers/v1/sources.py +3 -3
- letta/server/server.py +0 -3
- letta/services/agent_manager.py +194 -147
- letta/services/block_manager.py +18 -18
- letta/services/context_window_calculator/context_window_calculator.py +15 -10
- letta/services/context_window_calculator/token_counter.py +40 -0
- letta/services/file_manager.py +37 -0
- letta/services/file_processor/chunker/line_chunker.py +17 -0
- letta/services/file_processor/embedder/openai_embedder.py +50 -5
- letta/services/files_agents_manager.py +12 -2
- letta/services/group_manager.py +11 -11
- letta/services/source_manager.py +19 -3
- letta/services/tool_executor/core_tool_executor.py +2 -2
- letta/services/tool_executor/files_tool_executor.py +6 -1
- {letta_nightly-0.8.13.dev20250714104447.dist-info → letta_nightly-0.8.15.dev20250715080149.dist-info}/METADATA +1 -1
- {letta_nightly-0.8.13.dev20250714104447.dist-info → letta_nightly-0.8.15.dev20250715080149.dist-info}/RECORD +36 -36
- {letta_nightly-0.8.13.dev20250714104447.dist-info → letta_nightly-0.8.15.dev20250715080149.dist-info}/LICENSE +0 -0
- {letta_nightly-0.8.13.dev20250714104447.dist-info → letta_nightly-0.8.15.dev20250715080149.dist-info}/WHEEL +0 -0
- {letta_nightly-0.8.13.dev20250714104447.dist-info → letta_nightly-0.8.15.dev20250715080149.dist-info}/entry_points.txt +0 -0
letta/__init__.py
CHANGED
letta/constants.py
CHANGED
|
@@ -372,3 +372,9 @@ PINECONE_METRIC = "cosine"
|
|
|
372
372
|
PINECONE_CLOUD = "aws"
|
|
373
373
|
PINECONE_REGION = "us-east-1"
|
|
374
374
|
PINECONE_MAX_BATCH_SIZE = 96
|
|
375
|
+
|
|
376
|
+
# retry configuration
|
|
377
|
+
PINECONE_MAX_RETRY_ATTEMPTS = 3
|
|
378
|
+
PINECONE_RETRY_BASE_DELAY = 1.0 # seconds
|
|
379
|
+
PINECONE_RETRY_MAX_DELAY = 60.0 # seconds
|
|
380
|
+
PINECONE_RETRY_BACKOFF_FACTOR = 2.0
|
|
@@ -135,7 +135,7 @@ def core_memory_append(agent_state: "AgentState", label: str, content: str) -> O
|
|
|
135
135
|
Append to the contents of core memory.
|
|
136
136
|
|
|
137
137
|
Args:
|
|
138
|
-
label (str): Section of the memory to be edited
|
|
138
|
+
label (str): Section of the memory to be edited.
|
|
139
139
|
content (str): Content to write to the memory. All unicode (including emojis) are supported.
|
|
140
140
|
|
|
141
141
|
Returns:
|
|
@@ -152,7 +152,7 @@ def core_memory_replace(agent_state: "AgentState", label: str, old_content: str,
|
|
|
152
152
|
Replace the contents of core memory. To delete memories, use an empty string for new_content.
|
|
153
153
|
|
|
154
154
|
Args:
|
|
155
|
-
label (str): Section of the memory to be edited
|
|
155
|
+
label (str): Section of the memory to be edited.
|
|
156
156
|
old_content (str): String to replace. Must be an exact match.
|
|
157
157
|
new_content (str): Content to write to the memory. All unicode (including emojis) are supported.
|
|
158
158
|
|
|
@@ -46,12 +46,12 @@ async def grep_files(
|
|
|
46
46
|
context_lines: Optional[int] = 3,
|
|
47
47
|
) -> str:
|
|
48
48
|
"""
|
|
49
|
-
|
|
49
|
+
Searches file contents for pattern matches with surrounding context.
|
|
50
50
|
|
|
51
|
-
|
|
52
|
-
-
|
|
53
|
-
-
|
|
54
|
-
-
|
|
51
|
+
Ideal for:
|
|
52
|
+
- Finding specific code elements (variables, functions, keywords)
|
|
53
|
+
- Locating error messages or specific text across multiple files
|
|
54
|
+
- Examining code in context to understand usage patterns
|
|
55
55
|
|
|
56
56
|
Args:
|
|
57
57
|
pattern (str): Keyword or regex pattern to search within file contents.
|
|
@@ -67,15 +67,15 @@ async def grep_files(
|
|
|
67
67
|
|
|
68
68
|
async def semantic_search_files(agent_state: "AgentState", query: str, limit: int = 5) -> List["FileMetadata"]:
|
|
69
69
|
"""
|
|
70
|
-
|
|
70
|
+
Searches file contents using semantic meaning rather than exact matches.
|
|
71
71
|
|
|
72
|
-
|
|
73
|
-
-
|
|
74
|
-
-
|
|
75
|
-
-
|
|
72
|
+
Ideal for:
|
|
73
|
+
- Finding conceptually related information across files
|
|
74
|
+
- Discovering relevant content without knowing exact keywords
|
|
75
|
+
- Locating files with similar topics or themes
|
|
76
76
|
|
|
77
77
|
Args:
|
|
78
|
-
query (str): The search query.
|
|
78
|
+
query (str): The search query text to find semantically similar content.
|
|
79
79
|
limit: Maximum number of results to return (default: 5)
|
|
80
80
|
|
|
81
81
|
Returns:
|
letta/helpers/decorators.py
CHANGED
|
@@ -152,7 +152,7 @@ def async_redis_cache(
|
|
|
152
152
|
def get_cache_key(*args, **kwargs):
|
|
153
153
|
return f"{prefix}:{key_func(*args, **kwargs)}"
|
|
154
154
|
|
|
155
|
-
|
|
155
|
+
async_wrapper.cache_invalidate = invalidate
|
|
156
156
|
async_wrapper.cache_key_func = get_cache_key
|
|
157
157
|
async_wrapper.cache_stats = stats
|
|
158
158
|
return async_wrapper
|
letta/helpers/pinecone_utils.py
CHANGED
|
@@ -1,8 +1,20 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import random
|
|
3
|
+
import time
|
|
4
|
+
from functools import wraps
|
|
1
5
|
from typing import Any, Dict, List
|
|
2
6
|
|
|
7
|
+
from letta.otel.tracing import trace_method
|
|
8
|
+
|
|
3
9
|
try:
|
|
4
10
|
from pinecone import IndexEmbed, PineconeAsyncio
|
|
5
|
-
from pinecone.exceptions.exceptions import
|
|
11
|
+
from pinecone.exceptions.exceptions import (
|
|
12
|
+
ForbiddenException,
|
|
13
|
+
NotFoundException,
|
|
14
|
+
PineconeApiException,
|
|
15
|
+
ServiceException,
|
|
16
|
+
UnauthorizedException,
|
|
17
|
+
)
|
|
6
18
|
|
|
7
19
|
PINECONE_AVAILABLE = True
|
|
8
20
|
except ImportError:
|
|
@@ -12,8 +24,12 @@ from letta.constants import (
|
|
|
12
24
|
PINECONE_CLOUD,
|
|
13
25
|
PINECONE_EMBEDDING_MODEL,
|
|
14
26
|
PINECONE_MAX_BATCH_SIZE,
|
|
27
|
+
PINECONE_MAX_RETRY_ATTEMPTS,
|
|
15
28
|
PINECONE_METRIC,
|
|
16
29
|
PINECONE_REGION,
|
|
30
|
+
PINECONE_RETRY_BACKOFF_FACTOR,
|
|
31
|
+
PINECONE_RETRY_BASE_DELAY,
|
|
32
|
+
PINECONE_RETRY_MAX_DELAY,
|
|
17
33
|
PINECONE_TEXT_FIELD_NAME,
|
|
18
34
|
)
|
|
19
35
|
from letta.log import get_logger
|
|
@@ -23,6 +39,87 @@ from letta.settings import settings
|
|
|
23
39
|
logger = get_logger(__name__)
|
|
24
40
|
|
|
25
41
|
|
|
42
|
+
def pinecone_retry(
|
|
43
|
+
max_attempts: int = PINECONE_MAX_RETRY_ATTEMPTS,
|
|
44
|
+
base_delay: float = PINECONE_RETRY_BASE_DELAY,
|
|
45
|
+
max_delay: float = PINECONE_RETRY_MAX_DELAY,
|
|
46
|
+
backoff_factor: float = PINECONE_RETRY_BACKOFF_FACTOR,
|
|
47
|
+
):
|
|
48
|
+
"""
|
|
49
|
+
Decorator to retry Pinecone operations with exponential backoff.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
max_attempts: Maximum number of retry attempts
|
|
53
|
+
base_delay: Base delay in seconds for the first retry
|
|
54
|
+
max_delay: Maximum delay in seconds between retries
|
|
55
|
+
backoff_factor: Factor to increase delay after each failed attempt
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
def decorator(func):
|
|
59
|
+
@wraps(func)
|
|
60
|
+
async def wrapper(*args, **kwargs):
|
|
61
|
+
operation_name = func.__name__
|
|
62
|
+
start_time = time.time()
|
|
63
|
+
|
|
64
|
+
for attempt in range(max_attempts):
|
|
65
|
+
try:
|
|
66
|
+
logger.debug(f"[Pinecone] Starting {operation_name} (attempt {attempt + 1}/{max_attempts})")
|
|
67
|
+
result = await func(*args, **kwargs)
|
|
68
|
+
|
|
69
|
+
execution_time = time.time() - start_time
|
|
70
|
+
logger.info(f"[Pinecone] {operation_name} completed successfully in {execution_time:.2f}s")
|
|
71
|
+
return result
|
|
72
|
+
|
|
73
|
+
except (ServiceException, PineconeApiException) as e:
|
|
74
|
+
# retryable server errors
|
|
75
|
+
if attempt == max_attempts - 1:
|
|
76
|
+
execution_time = time.time() - start_time
|
|
77
|
+
logger.error(f"[Pinecone] {operation_name} failed after {max_attempts} attempts in {execution_time:.2f}s: {str(e)}")
|
|
78
|
+
raise
|
|
79
|
+
|
|
80
|
+
# calculate delay with exponential backoff and jitter
|
|
81
|
+
delay = min(base_delay * (backoff_factor**attempt), max_delay)
|
|
82
|
+
jitter = random.uniform(0, delay * 0.1) # add up to 10% jitter
|
|
83
|
+
total_delay = delay + jitter
|
|
84
|
+
|
|
85
|
+
logger.warning(
|
|
86
|
+
f"[Pinecone] {operation_name} failed (attempt {attempt + 1}/{max_attempts}): {str(e)}. Retrying in {total_delay:.2f}s"
|
|
87
|
+
)
|
|
88
|
+
await asyncio.sleep(total_delay)
|
|
89
|
+
|
|
90
|
+
except (UnauthorizedException, ForbiddenException) as e:
|
|
91
|
+
# non-retryable auth errors
|
|
92
|
+
execution_time = time.time() - start_time
|
|
93
|
+
logger.error(f"[Pinecone] {operation_name} failed with auth error in {execution_time:.2f}s: {str(e)}")
|
|
94
|
+
raise
|
|
95
|
+
|
|
96
|
+
except NotFoundException as e:
|
|
97
|
+
# non-retryable not found errors
|
|
98
|
+
execution_time = time.time() - start_time
|
|
99
|
+
logger.warning(f"[Pinecone] {operation_name} failed with not found error in {execution_time:.2f}s: {str(e)}")
|
|
100
|
+
raise
|
|
101
|
+
|
|
102
|
+
except Exception as e:
|
|
103
|
+
# other unexpected errors - retry once then fail
|
|
104
|
+
if attempt == max_attempts - 1:
|
|
105
|
+
execution_time = time.time() - start_time
|
|
106
|
+
logger.error(f"[Pinecone] {operation_name} failed after {max_attempts} attempts in {execution_time:.2f}s: {str(e)}")
|
|
107
|
+
raise
|
|
108
|
+
|
|
109
|
+
delay = min(base_delay * (backoff_factor**attempt), max_delay)
|
|
110
|
+
jitter = random.uniform(0, delay * 0.1)
|
|
111
|
+
total_delay = delay + jitter
|
|
112
|
+
|
|
113
|
+
logger.warning(
|
|
114
|
+
f"[Pinecone] {operation_name} failed with unexpected error (attempt {attempt + 1}/{max_attempts}): {str(e)}. Retrying in {total_delay:.2f}s"
|
|
115
|
+
)
|
|
116
|
+
await asyncio.sleep(total_delay)
|
|
117
|
+
|
|
118
|
+
return wrapper
|
|
119
|
+
|
|
120
|
+
return decorator
|
|
121
|
+
|
|
122
|
+
|
|
26
123
|
def should_use_pinecone(verbose: bool = False):
|
|
27
124
|
if verbose:
|
|
28
125
|
logger.info(
|
|
@@ -44,29 +141,42 @@ def should_use_pinecone(verbose: bool = False):
|
|
|
44
141
|
)
|
|
45
142
|
|
|
46
143
|
|
|
144
|
+
@pinecone_retry()
|
|
145
|
+
@trace_method
|
|
47
146
|
async def upsert_pinecone_indices():
|
|
48
147
|
if not PINECONE_AVAILABLE:
|
|
49
148
|
raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
|
|
50
149
|
|
|
51
|
-
|
|
150
|
+
indices = get_pinecone_indices()
|
|
151
|
+
logger.info(f"[Pinecone] Upserting {len(indices)} indices: {indices}")
|
|
152
|
+
|
|
153
|
+
for index_name in indices:
|
|
52
154
|
async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
|
|
53
155
|
if not await pc.has_index(index_name):
|
|
156
|
+
logger.info(f"[Pinecone] Creating index {index_name} with model {PINECONE_EMBEDDING_MODEL}")
|
|
54
157
|
await pc.create_index_for_model(
|
|
55
158
|
name=index_name,
|
|
56
159
|
cloud=PINECONE_CLOUD,
|
|
57
160
|
region=PINECONE_REGION,
|
|
58
161
|
embed=IndexEmbed(model=PINECONE_EMBEDDING_MODEL, field_map={"text": PINECONE_TEXT_FIELD_NAME}, metric=PINECONE_METRIC),
|
|
59
162
|
)
|
|
163
|
+
logger.info(f"[Pinecone] Successfully created index {index_name}")
|
|
164
|
+
else:
|
|
165
|
+
logger.debug(f"[Pinecone] Index {index_name} already exists")
|
|
60
166
|
|
|
61
167
|
|
|
62
168
|
def get_pinecone_indices() -> List[str]:
|
|
63
169
|
return [settings.pinecone_agent_index, settings.pinecone_source_index]
|
|
64
170
|
|
|
65
171
|
|
|
172
|
+
@pinecone_retry()
|
|
173
|
+
@trace_method
|
|
66
174
|
async def upsert_file_records_to_pinecone_index(file_id: str, source_id: str, chunks: List[str], actor: User):
|
|
67
175
|
if not PINECONE_AVAILABLE:
|
|
68
176
|
raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
|
|
69
177
|
|
|
178
|
+
logger.info(f"[Pinecone] Preparing to upsert {len(chunks)} chunks for file {file_id} source {source_id}")
|
|
179
|
+
|
|
70
180
|
records = []
|
|
71
181
|
for i, chunk in enumerate(chunks):
|
|
72
182
|
record = {
|
|
@@ -77,14 +187,19 @@ async def upsert_file_records_to_pinecone_index(file_id: str, source_id: str, ch
|
|
|
77
187
|
}
|
|
78
188
|
records.append(record)
|
|
79
189
|
|
|
190
|
+
logger.debug(f"[Pinecone] Created {len(records)} records for file {file_id}")
|
|
80
191
|
return await upsert_records_to_pinecone_index(records, actor)
|
|
81
192
|
|
|
82
193
|
|
|
194
|
+
@pinecone_retry()
|
|
195
|
+
@trace_method
|
|
83
196
|
async def delete_file_records_from_pinecone_index(file_id: str, actor: User):
|
|
84
197
|
if not PINECONE_AVAILABLE:
|
|
85
198
|
raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
|
|
86
199
|
|
|
87
200
|
namespace = actor.organization_id
|
|
201
|
+
logger.info(f"[Pinecone] Deleting records for file {file_id} from index {settings.pinecone_source_index} namespace {namespace}")
|
|
202
|
+
|
|
88
203
|
try:
|
|
89
204
|
async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
|
|
90
205
|
description = await pc.describe_index(name=settings.pinecone_source_index)
|
|
@@ -95,48 +210,72 @@ async def delete_file_records_from_pinecone_index(file_id: str, actor: User):
|
|
|
95
210
|
},
|
|
96
211
|
namespace=namespace,
|
|
97
212
|
)
|
|
213
|
+
logger.info(f"[Pinecone] Successfully deleted records for file {file_id}")
|
|
98
214
|
except NotFoundException:
|
|
99
|
-
logger.warning(f"Pinecone
|
|
215
|
+
logger.warning(f"[Pinecone] Namespace {namespace} not found for file {file_id} and org {actor.organization_id}")
|
|
100
216
|
|
|
101
217
|
|
|
218
|
+
@pinecone_retry()
|
|
219
|
+
@trace_method
|
|
102
220
|
async def delete_source_records_from_pinecone_index(source_id: str, actor: User):
|
|
103
221
|
if not PINECONE_AVAILABLE:
|
|
104
222
|
raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
|
|
105
223
|
|
|
106
224
|
namespace = actor.organization_id
|
|
225
|
+
logger.info(f"[Pinecone] Deleting records for source {source_id} from index {settings.pinecone_source_index} namespace {namespace}")
|
|
226
|
+
|
|
107
227
|
try:
|
|
108
228
|
async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
|
|
109
229
|
description = await pc.describe_index(name=settings.pinecone_source_index)
|
|
110
230
|
async with pc.IndexAsyncio(host=description.index.host) as dense_index:
|
|
111
231
|
await dense_index.delete(filter={"source_id": {"$eq": source_id}}, namespace=namespace)
|
|
232
|
+
logger.info(f"[Pinecone] Successfully deleted records for source {source_id}")
|
|
112
233
|
except NotFoundException:
|
|
113
|
-
logger.warning(f"Pinecone
|
|
234
|
+
logger.warning(f"[Pinecone] Namespace {namespace} not found for source {source_id} and org {actor.organization_id}")
|
|
114
235
|
|
|
115
236
|
|
|
237
|
+
@pinecone_retry()
|
|
238
|
+
@trace_method
|
|
116
239
|
async def upsert_records_to_pinecone_index(records: List[dict], actor: User):
|
|
117
240
|
if not PINECONE_AVAILABLE:
|
|
118
241
|
raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
|
|
119
242
|
|
|
243
|
+
logger.info(f"[Pinecone] Upserting {len(records)} records to index {settings.pinecone_source_index} for org {actor.organization_id}")
|
|
244
|
+
|
|
120
245
|
async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
|
|
121
246
|
description = await pc.describe_index(name=settings.pinecone_source_index)
|
|
122
247
|
async with pc.IndexAsyncio(host=description.index.host) as dense_index:
|
|
123
|
-
#
|
|
248
|
+
# process records in batches to avoid exceeding pinecone limits
|
|
249
|
+
total_batches = (len(records) + PINECONE_MAX_BATCH_SIZE - 1) // PINECONE_MAX_BATCH_SIZE
|
|
250
|
+
logger.debug(f"[Pinecone] Processing {total_batches} batches of max {PINECONE_MAX_BATCH_SIZE} records each")
|
|
251
|
+
|
|
124
252
|
for i in range(0, len(records), PINECONE_MAX_BATCH_SIZE):
|
|
125
253
|
batch = records[i : i + PINECONE_MAX_BATCH_SIZE]
|
|
254
|
+
batch_num = (i // PINECONE_MAX_BATCH_SIZE) + 1
|
|
255
|
+
|
|
256
|
+
logger.debug(f"[Pinecone] Upserting batch {batch_num}/{total_batches} with {len(batch)} records")
|
|
126
257
|
await dense_index.upsert_records(actor.organization_id, batch)
|
|
127
258
|
|
|
259
|
+
logger.info(f"[Pinecone] Successfully upserted all {len(records)} records in {total_batches} batches")
|
|
260
|
+
|
|
128
261
|
|
|
262
|
+
@pinecone_retry()
|
|
263
|
+
@trace_method
|
|
129
264
|
async def search_pinecone_index(query: str, limit: int, filter: Dict[str, Any], actor: User) -> Dict[str, Any]:
|
|
130
265
|
if not PINECONE_AVAILABLE:
|
|
131
266
|
raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
|
|
132
267
|
|
|
268
|
+
namespace = actor.organization_id
|
|
269
|
+
logger.info(
|
|
270
|
+
f"[Pinecone] Searching index {settings.pinecone_source_index} namespace {namespace} with query length {len(query)} chars, limit {limit}"
|
|
271
|
+
)
|
|
272
|
+
logger.debug(f"[Pinecone] Search filter: {filter}")
|
|
273
|
+
|
|
133
274
|
async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
|
|
134
275
|
description = await pc.describe_index(name=settings.pinecone_source_index)
|
|
135
276
|
async with pc.IndexAsyncio(host=description.index.host) as dense_index:
|
|
136
|
-
namespace = actor.organization_id
|
|
137
|
-
|
|
138
277
|
try:
|
|
139
|
-
#
|
|
278
|
+
# search the dense index with reranking
|
|
140
279
|
search_results = await dense_index.search(
|
|
141
280
|
namespace=namespace,
|
|
142
281
|
query={
|
|
@@ -146,17 +285,26 @@ async def search_pinecone_index(query: str, limit: int, filter: Dict[str, Any],
|
|
|
146
285
|
},
|
|
147
286
|
rerank={"model": "bge-reranker-v2-m3", "top_n": limit, "rank_fields": [PINECONE_TEXT_FIELD_NAME]},
|
|
148
287
|
)
|
|
288
|
+
|
|
289
|
+
result_count = len(search_results.get("matches", []))
|
|
290
|
+
logger.info(f"[Pinecone] Search completed, found {result_count} matches")
|
|
149
291
|
return search_results
|
|
292
|
+
|
|
150
293
|
except Exception as e:
|
|
151
|
-
logger.warning(f"Failed to search
|
|
294
|
+
logger.warning(f"[Pinecone] Failed to search namespace {namespace}: {str(e)}")
|
|
152
295
|
raise e
|
|
153
296
|
|
|
154
297
|
|
|
298
|
+
@pinecone_retry()
|
|
299
|
+
@trace_method
|
|
155
300
|
async def list_pinecone_index_for_files(file_id: str, actor: User, limit: int = None, pagination_token: str = None) -> List[str]:
|
|
156
301
|
if not PINECONE_AVAILABLE:
|
|
157
302
|
raise ImportError("Pinecone is not available. Please install pinecone to use this feature.")
|
|
158
303
|
|
|
159
304
|
namespace = actor.organization_id
|
|
305
|
+
logger.info(f"[Pinecone] Listing records for file {file_id} from index {settings.pinecone_source_index} namespace {namespace}")
|
|
306
|
+
logger.debug(f"[Pinecone] List params - limit: {limit}, pagination_token: {pagination_token}")
|
|
307
|
+
|
|
160
308
|
try:
|
|
161
309
|
async with PineconeAsyncio(api_key=settings.pinecone_api_key) as pc:
|
|
162
310
|
description = await pc.describe_index(name=settings.pinecone_source_index)
|
|
@@ -172,9 +320,14 @@ async def list_pinecone_index_for_files(file_id: str, actor: User, limit: int =
|
|
|
172
320
|
result = []
|
|
173
321
|
async for ids in dense_index.list(**kwargs):
|
|
174
322
|
result.extend(ids)
|
|
323
|
+
|
|
324
|
+
logger.info(f"[Pinecone] Successfully listed {len(result)} records for file {file_id}")
|
|
175
325
|
return result
|
|
326
|
+
|
|
176
327
|
except Exception as e:
|
|
177
|
-
logger.warning(f"Failed to list
|
|
328
|
+
logger.warning(f"[Pinecone] Failed to list records for file {file_id} in namespace {namespace}: {str(e)}")
|
|
178
329
|
raise e
|
|
330
|
+
|
|
179
331
|
except NotFoundException:
|
|
180
|
-
logger.warning(f"Pinecone
|
|
332
|
+
logger.warning(f"[Pinecone] Namespace {namespace} not found for file {file_id} and org {actor.organization_id}")
|
|
333
|
+
return []
|
letta/orm/agent.py
CHANGED
|
@@ -314,7 +314,7 @@ class Agent(SqlalchemyBase, OrganizationMixin, AsyncAttrs):
|
|
|
314
314
|
state["sources"] = [s.to_pydantic() for s in sources]
|
|
315
315
|
state["memory"] = Memory(
|
|
316
316
|
blocks=[m.to_pydantic() for m in memory],
|
|
317
|
-
file_blocks=[block for b in
|
|
317
|
+
file_blocks=[block for b in file_agents if (block := b.to_pydantic_block()) is not None],
|
|
318
318
|
prompt_template=get_prompt_template_for_agent_type(self.agent_type),
|
|
319
319
|
)
|
|
320
320
|
state["identity_ids"] = [i.id for i in identities]
|
letta/orm/file.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import uuid
|
|
2
|
-
from typing import TYPE_CHECKING,
|
|
2
|
+
from typing import TYPE_CHECKING, Optional
|
|
3
3
|
|
|
4
4
|
from sqlalchemy import ForeignKey, Index, Integer, String, Text, UniqueConstraint, desc
|
|
5
5
|
from sqlalchemy.ext.asyncio import AsyncAttrs
|
|
@@ -11,10 +11,7 @@ from letta.schemas.enums import FileProcessingStatus
|
|
|
11
11
|
from letta.schemas.file import FileMetadata as PydanticFileMetadata
|
|
12
12
|
|
|
13
13
|
if TYPE_CHECKING:
|
|
14
|
-
|
|
15
|
-
from letta.orm.organization import Organization
|
|
16
|
-
from letta.orm.passage import SourcePassage
|
|
17
|
-
from letta.orm.source import Source
|
|
14
|
+
pass
|
|
18
15
|
|
|
19
16
|
|
|
20
17
|
# TODO: Note that this is NOT organization scoped, this is potentially dangerous if we misuse this
|
|
@@ -64,18 +61,6 @@ class FileMetadata(SqlalchemyBase, OrganizationMixin, SourceMixin, AsyncAttrs):
|
|
|
64
61
|
chunks_embedded: Mapped[Optional[int]] = mapped_column(Integer, nullable=True, doc="Number of chunks that have been embedded.")
|
|
65
62
|
|
|
66
63
|
# relationships
|
|
67
|
-
organization: Mapped["Organization"] = relationship("Organization", back_populates="files", lazy="selectin")
|
|
68
|
-
source: Mapped["Source"] = relationship("Source", back_populates="files", lazy="selectin")
|
|
69
|
-
source_passages: Mapped[List["SourcePassage"]] = relationship(
|
|
70
|
-
"SourcePassage", back_populates="file", lazy="selectin", cascade="all, delete-orphan"
|
|
71
|
-
)
|
|
72
|
-
file_agents: Mapped[List["FileAgent"]] = relationship(
|
|
73
|
-
"FileAgent",
|
|
74
|
-
back_populates="file",
|
|
75
|
-
lazy="selectin",
|
|
76
|
-
cascade="all, delete-orphan",
|
|
77
|
-
passive_deletes=True, # ← add this
|
|
78
|
-
)
|
|
79
64
|
content: Mapped[Optional["FileContent"]] = relationship(
|
|
80
65
|
"FileContent",
|
|
81
66
|
uselist=False,
|
letta/orm/files_agents.py
CHANGED
|
@@ -12,7 +12,7 @@ from letta.schemas.block import Block as PydanticBlock
|
|
|
12
12
|
from letta.schemas.file import FileAgent as PydanticFileAgent
|
|
13
13
|
|
|
14
14
|
if TYPE_CHECKING:
|
|
15
|
-
|
|
15
|
+
pass
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
class FileAgent(SqlalchemyBase, OrganizationMixin):
|
|
@@ -55,6 +55,12 @@ class FileAgent(SqlalchemyBase, OrganizationMixin):
|
|
|
55
55
|
nullable=False,
|
|
56
56
|
doc="ID of the agent",
|
|
57
57
|
)
|
|
58
|
+
source_id: Mapped[str] = mapped_column(
|
|
59
|
+
String,
|
|
60
|
+
ForeignKey("sources.id", ondelete="CASCADE"),
|
|
61
|
+
nullable=False,
|
|
62
|
+
doc="ID of the source (denormalized from files.source_id)",
|
|
63
|
+
)
|
|
58
64
|
|
|
59
65
|
file_name: Mapped[str] = mapped_column(
|
|
60
66
|
String,
|
|
@@ -78,13 +84,6 @@ class FileAgent(SqlalchemyBase, OrganizationMixin):
|
|
|
78
84
|
back_populates="file_agents",
|
|
79
85
|
lazy="selectin",
|
|
80
86
|
)
|
|
81
|
-
file: Mapped["FileMetadata"] = relationship(
|
|
82
|
-
"FileMetadata",
|
|
83
|
-
foreign_keys=[file_id],
|
|
84
|
-
lazy="selectin",
|
|
85
|
-
back_populates="file_agents",
|
|
86
|
-
passive_deletes=True, # ← add this
|
|
87
|
-
)
|
|
88
87
|
|
|
89
88
|
# TODO: This is temporary as we figure out if we want FileBlock as a first class citizen
|
|
90
89
|
def to_pydantic_block(self) -> PydanticBlock:
|
|
@@ -99,8 +98,8 @@ class FileAgent(SqlalchemyBase, OrganizationMixin):
|
|
|
99
98
|
return PydanticBlock(
|
|
100
99
|
organization_id=self.organization_id,
|
|
101
100
|
value=visible_content,
|
|
102
|
-
label=self.file.file_name
|
|
101
|
+
label=self.file_name, # use denormalized file_name instead of self.file.file_name
|
|
103
102
|
read_only=True,
|
|
104
|
-
metadata={"source_id": self.
|
|
103
|
+
metadata={"source_id": self.source_id}, # use denormalized source_id
|
|
105
104
|
limit=CORE_MEMORY_SOURCE_CHAR_LIMIT,
|
|
106
105
|
)
|
letta/orm/organization.py
CHANGED
|
@@ -9,7 +9,6 @@ if TYPE_CHECKING:
|
|
|
9
9
|
from letta.orm.agent import Agent
|
|
10
10
|
from letta.orm.agent_passage import AgentPassage
|
|
11
11
|
from letta.orm.block import Block
|
|
12
|
-
from letta.orm.file import FileMetadata
|
|
13
12
|
from letta.orm.group import Group
|
|
14
13
|
from letta.orm.identity import Identity
|
|
15
14
|
from letta.orm.llm_batch_item import LLMBatchItem
|
|
@@ -18,7 +17,6 @@ if TYPE_CHECKING:
|
|
|
18
17
|
from letta.orm.provider import Provider
|
|
19
18
|
from letta.orm.sandbox_config import AgentEnvironmentVariable, SandboxConfig
|
|
20
19
|
from letta.orm.sandbox_environment_variable import SandboxEnvironmentVariable
|
|
21
|
-
from letta.orm.source import Source
|
|
22
20
|
from letta.orm.source_passage import SourcePassage
|
|
23
21
|
from letta.orm.tool import Tool
|
|
24
22
|
from letta.orm.user import User
|
|
@@ -38,8 +36,6 @@ class Organization(SqlalchemyBase):
|
|
|
38
36
|
tools: Mapped[List["Tool"]] = relationship("Tool", back_populates="organization", cascade="all, delete-orphan")
|
|
39
37
|
# mcp_servers: Mapped[List["MCPServer"]] = relationship("MCPServer", back_populates="organization", cascade="all, delete-orphan")
|
|
40
38
|
blocks: Mapped[List["Block"]] = relationship("Block", back_populates="organization", cascade="all, delete-orphan")
|
|
41
|
-
sources: Mapped[List["Source"]] = relationship("Source", back_populates="organization", cascade="all, delete-orphan")
|
|
42
|
-
files: Mapped[List["FileMetadata"]] = relationship("FileMetadata", back_populates="organization", cascade="all, delete-orphan")
|
|
43
39
|
sandbox_configs: Mapped[List["SandboxConfig"]] = relationship(
|
|
44
40
|
"SandboxConfig", back_populates="organization", cascade="all, delete-orphan"
|
|
45
41
|
)
|
letta/orm/passage.py
CHANGED
|
@@ -49,11 +49,6 @@ class SourcePassage(BasePassage, FileMixin, SourceMixin):
|
|
|
49
49
|
|
|
50
50
|
file_name: Mapped[str] = mapped_column(doc="The name of the file that this passage was derived from")
|
|
51
51
|
|
|
52
|
-
@declared_attr
|
|
53
|
-
def file(cls) -> Mapped["FileMetadata"]:
|
|
54
|
-
"""Relationship to file"""
|
|
55
|
-
return relationship("FileMetadata", back_populates="source_passages", lazy="selectin")
|
|
56
|
-
|
|
57
52
|
@declared_attr
|
|
58
53
|
def organization(cls) -> Mapped["Organization"]:
|
|
59
54
|
return relationship("Organization", back_populates="source_passages", lazy="selectin")
|
|
@@ -74,11 +69,6 @@ class SourcePassage(BasePassage, FileMixin, SourceMixin):
|
|
|
74
69
|
{"extend_existing": True},
|
|
75
70
|
)
|
|
76
71
|
|
|
77
|
-
@declared_attr
|
|
78
|
-
def source(cls) -> Mapped["Source"]:
|
|
79
|
-
"""Relationship to source"""
|
|
80
|
-
return relationship("Source", back_populates="passages", lazy="selectin", passive_deletes=True)
|
|
81
|
-
|
|
82
72
|
|
|
83
73
|
class AgentPassage(BasePassage, AgentMixin):
|
|
84
74
|
"""Passages created by agents as archival memories"""
|
letta/orm/source.py
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
from typing import TYPE_CHECKING,
|
|
1
|
+
from typing import TYPE_CHECKING, Optional
|
|
2
2
|
|
|
3
3
|
from sqlalchemy import JSON, Index, UniqueConstraint
|
|
4
|
-
from sqlalchemy.orm import Mapped, mapped_column
|
|
4
|
+
from sqlalchemy.orm import Mapped, mapped_column
|
|
5
5
|
|
|
6
|
-
from letta.orm import FileMetadata
|
|
7
6
|
from letta.orm.custom_columns import EmbeddingConfigColumn
|
|
8
7
|
from letta.orm.mixins import OrganizationMixin
|
|
9
8
|
from letta.orm.sqlalchemy_base import SqlalchemyBase
|
|
@@ -11,10 +10,7 @@ from letta.schemas.embedding_config import EmbeddingConfig
|
|
|
11
10
|
from letta.schemas.source import Source as PydanticSource
|
|
12
11
|
|
|
13
12
|
if TYPE_CHECKING:
|
|
14
|
-
|
|
15
|
-
from letta.orm.file import FileMetadata
|
|
16
|
-
from letta.orm.organization import Organization
|
|
17
|
-
from letta.orm.passage import SourcePassage
|
|
13
|
+
pass
|
|
18
14
|
|
|
19
15
|
|
|
20
16
|
class Source(SqlalchemyBase, OrganizationMixin):
|
|
@@ -34,16 +30,3 @@ class Source(SqlalchemyBase, OrganizationMixin):
|
|
|
34
30
|
instructions: Mapped[str] = mapped_column(nullable=True, doc="instructions for how to use the source")
|
|
35
31
|
embedding_config: Mapped[EmbeddingConfig] = mapped_column(EmbeddingConfigColumn, doc="Configuration settings for embedding.")
|
|
36
32
|
metadata_: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True, doc="metadata for the source.")
|
|
37
|
-
|
|
38
|
-
# relationships
|
|
39
|
-
organization: Mapped["Organization"] = relationship("Organization", back_populates="sources")
|
|
40
|
-
files: Mapped[List["FileMetadata"]] = relationship("FileMetadata", back_populates="source", cascade="all, delete-orphan")
|
|
41
|
-
passages: Mapped[List["SourcePassage"]] = relationship("SourcePassage", back_populates="source", cascade="all, delete-orphan")
|
|
42
|
-
agents: Mapped[List["Agent"]] = relationship(
|
|
43
|
-
"Agent",
|
|
44
|
-
secondary="sources_agents",
|
|
45
|
-
back_populates="sources",
|
|
46
|
-
lazy="selectin",
|
|
47
|
-
cascade="save-update", # Only propagate save and update operations
|
|
48
|
-
passive_deletes=True, # Let the database handle deletions
|
|
49
|
-
)
|