letta-nightly 0.11.4.dev20250826104242__py3-none-any.whl → 0.11.6.dev20250827050912__py3-none-any.whl

letta/server/rest_api/redis_stream_manager.py

@@ -0,0 +1,300 @@
+"""Redis stream manager for reading and writing SSE chunks with batching and TTL."""
+
+import asyncio
+import json
+import time
+from collections import defaultdict
+from typing import AsyncIterator, Dict, List, Optional
+
+from letta.data_sources.redis_client import AsyncRedisClient
+from letta.log import get_logger
+
+logger = get_logger(__name__)
+
+
+class RedisSSEStreamWriter:
+    """
+    Efficiently writes SSE chunks to Redis streams with batching and TTL management.
+
+    Features:
+    - Batches writes using Redis pipelines for performance
+    - Automatically sets/refreshes TTL on streams
+    - Tracks sequential IDs for cursor-based recovery
+    - Handles flush on size or time thresholds
+    """
+
+    def __init__(
+        self,
+        redis_client: AsyncRedisClient,
+        flush_interval: float = 0.5,
+        flush_size: int = 50,
+        stream_ttl_seconds: int = 10800,  # 3 hours default
+        max_stream_length: int = 10000,  # Max entries per stream
+    ):
+        """
+        Initialize the Redis SSE stream writer.
+
+        Args:
+            redis_client: Redis client instance
+            flush_interval: Seconds between automatic flushes
+            flush_size: Number of chunks to buffer before flushing
+            stream_ttl_seconds: TTL for streams in seconds (default: 6 hours)
+            max_stream_length: Maximum entries per stream before trimming
+        """
+        self.redis = redis_client
+        self.flush_interval = flush_interval
+        self.flush_size = flush_size
+        self.stream_ttl = stream_ttl_seconds
+        self.max_stream_length = max_stream_length
+
+        # Buffer for batching: run_id -> list of chunks
+        self.buffer: Dict[str, List[Dict]] = defaultdict(list)
+        # Track sequence IDs per run
+        self.seq_counters: Dict[str, int] = defaultdict(lambda: 1)
+        # Track last flush time per run
+        self.last_flush: Dict[str, float] = defaultdict(float)
+
+        # Background flush task
+        self._flush_task = None
+        self._running = False
+
+    async def start(self):
+        """Start the background flush task."""
+        if not self._running:
+            self._running = True
+            self._flush_task = asyncio.create_task(self._periodic_flush())
+
+    async def stop(self):
+        """Stop the background flush task and flush remaining data."""
+        self._running = False
+        if self._flush_task:
+            self._flush_task.cancel()
+            try:
+                await self._flush_task
+            except asyncio.CancelledError:
+                pass
+
+        for run_id in list(self.buffer.keys()):
+            if self.buffer[run_id]:
+                await self._flush_run(run_id)
+
+    async def write_chunk(
+        self,
+        run_id: str,
+        data: str,
+        is_complete: bool = False,
+    ) -> int:
+        """
+        Write an SSE chunk to the buffer for a specific run.
+
+        Args:
+            run_id: The run ID to write to
+            data: SSE-formatted chunk data
+            is_complete: Whether this is the final chunk
+
+        Returns:
+            The sequence ID assigned to this chunk
+        """
+        seq_id = self.seq_counters[run_id]
+        self.seq_counters[run_id] += 1
+
+        chunk = {
+            "seq_id": seq_id,
+            "data": data,
+            "timestamp": int(time.time() * 1000),
+        }
+
+        if is_complete:
+            chunk["complete"] = "true"
+
+        self.buffer[run_id].append(chunk)
+
+        should_flush = (
+            len(self.buffer[run_id]) >= self.flush_size or is_complete or (time.time() - self.last_flush[run_id]) > self.flush_interval
+        )
+
+        if should_flush:
+            await self._flush_run(run_id)
+
+        return seq_id
+
+    async def _flush_run(self, run_id: str):
+        """Flush buffered chunks for a specific run to Redis."""
+        if not self.buffer[run_id]:
+            return
+
+        chunks = self.buffer[run_id]
+        self.buffer[run_id] = []
+        stream_key = f"sse:run:{run_id}"
+
+        try:
+            client = await self.redis.get_client()
+
+            async with client.pipeline(transaction=False) as pipe:
+                for chunk in chunks:
+                    pipe.xadd(stream_key, chunk, maxlen=self.max_stream_length, approximate=True)
+
+                pipe.expire(stream_key, self.stream_ttl)
+
+                await pipe.execute()
+
+            self.last_flush[run_id] = time.time()
+
+            logger.debug(
+                f"Flushed {len(chunks)} chunks to Redis stream {stream_key}, " f"seq_ids {chunks[0]['seq_id']}-{chunks[-1]['seq_id']}"
+            )
+
+            if chunks[-1].get("complete") == "true":
+                self._cleanup_run(run_id)
+
+        except Exception as e:
+            logger.error(f"Failed to flush chunks for run {run_id}: {e}")
+            # Put chunks back in buffer to retry
+            self.buffer[run_id] = chunks + self.buffer[run_id]
+            raise
+
+    async def _periodic_flush(self):
+        """Background task to periodically flush buffers."""
+        while self._running:
+            try:
+                await asyncio.sleep(self.flush_interval)
+
+                # Check each run for time-based flush
+                current_time = time.time()
+                runs_to_flush = [
+                    run_id
+                    for run_id, last_flush in self.last_flush.items()
+                    if (current_time - last_flush) > self.flush_interval and self.buffer[run_id]
+                ]
+
+                for run_id in runs_to_flush:
+                    await self._flush_run(run_id)
+
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                logger.error(f"Error in periodic flush: {e}")
+
+    def _cleanup_run(self, run_id: str):
+        """Clean up tracking data for a completed run."""
+        self.buffer.pop(run_id, None)
+        self.seq_counters.pop(run_id, None)
+        self.last_flush.pop(run_id, None)
+
+    async def mark_complete(self, run_id: str):
+        """Mark a stream as complete and flush."""
+        # Add a [DONE] marker
+        await self.write_chunk(run_id, "data: [DONE]\n\n", is_complete=True)
+
+
+async def create_background_stream_processor(
+    stream_generator,
+    redis_client: AsyncRedisClient,
+    run_id: str,
+    writer: Optional[RedisSSEStreamWriter] = None,
+) -> None:
+    """
+    Process a stream in the background and store chunks to Redis.
+
+    This function consumes the stream generator and writes all chunks
+    to Redis for later retrieval.
+
+    Args:
+        stream_generator: The async generator yielding SSE chunks
+        redis_client: Redis client instance
+        run_id: The run ID to store chunks under
+        writer: Optional pre-configured writer (creates new if not provided)
+    """
+    if writer is None:
+        writer = RedisSSEStreamWriter(redis_client)
+        await writer.start()
+        should_stop_writer = True
+    else:
+        should_stop_writer = False
+
+    try:
+        async for chunk in stream_generator:
+            if isinstance(chunk, tuple):
+                chunk = chunk[0]
+
+            is_done = isinstance(chunk, str) and ("data: [DONE]" in chunk or "event: error" in chunk)
+
+            await writer.write_chunk(run_id=run_id, data=chunk, is_complete=is_done)
+
+            if is_done:
+                break
+
+    except Exception as e:
+        logger.error(f"Error processing stream for run {run_id}: {e}")
+        # Write error chunk
+        error_chunk = {"error": {"message": str(e)}}
+        await writer.write_chunk(run_id=run_id, data=f"event: error\ndata: {json.dumps(error_chunk)}\n\n", is_complete=True)
+    finally:
+        if should_stop_writer:
+            await writer.stop()
+
+
+async def redis_sse_stream_generator(
+    redis_client: AsyncRedisClient,
+    run_id: str,
+    starting_after: Optional[int] = None,
+    poll_interval: float = 0.1,
+    batch_size: int = 100,
+) -> AsyncIterator[str]:
+    """
+    Generate SSE events from Redis stream chunks.
+
+    This generator reads chunks stored in Redis streams and yields them as SSE events.
+    It supports cursor-based recovery by allowing you to start from a specific seq_id.
+
+    Args:
+        redis_client: Redis client instance
+        run_id: The run ID to read chunks for
+        starting_after: Sequential ID (integer) to start reading from (default: None for beginning)
+        poll_interval: Seconds to wait between polls when no new data (default: 0.1)
+        batch_size: Number of entries to read per batch (default: 100)
+
+    Yields:
+        SSE-formatted chunks from the Redis stream
+    """
+    stream_key = f"sse:run:{run_id}"
+    last_redis_id = "-"
+    cursor_seq_id = starting_after or 0
+
+    logger.debug(f"Starting redis_sse_stream_generator for run_id={run_id}, stream_key={stream_key}")
+
+    while True:
+        entries = await redis_client.xrange(stream_key, start=last_redis_id, count=batch_size)
+
+        if entries:
+            yielded_any = False
+            for entry_id, fields in entries:
+                if entry_id == last_redis_id:
+                    continue
+
+                chunk_seq_id = int(fields.get("seq_id", 0))
+                if chunk_seq_id > cursor_seq_id:
+                    data = fields.get("data", "")
+                    if not data:
+                        logger.debug(f"No data found for chunk {chunk_seq_id} in run {run_id}")
+                        continue
+
+                    if '"run_id":null' in data:
+                        data = data.replace('"run_id":null', f'"run_id":"{run_id}"')
+
+                    if '"seq_id":null' in data:
+                        data = data.replace('"seq_id":null', f'"seq_id":{chunk_seq_id}')
+
+                    yield data
+                    yielded_any = True
+
+                    if fields.get("complete") == "true":
+                        return
+
+                last_redis_id = entry_id
+
+            if not yielded_any and len(entries) > 1:
+                continue
+
+        if not entries or (len(entries) == 1 and entries[0][0] == last_redis_id):
+            await asyncio.sleep(poll_interval)

letta/server/rest_api/routers/v1/agents.py

@@ -14,7 +14,7 @@ from starlette.responses import Response, StreamingResponse
 
 from letta.agents.letta_agent import LettaAgent
 from letta.constants import AGENT_ID_PATTERN, DEFAULT_MAX_STEPS, DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG, REDIS_RUN_ID_PREFIX
-from letta.data_sources.redis_client import get_redis_client
+from letta.data_sources.redis_client import NoopAsyncRedisClient, get_redis_client
 from letta.errors import AgentExportIdMappingError, AgentExportProcessingError, AgentFileImportError, AgentNotFoundForExportError
 from letta.groups.sleeptime_multi_agent_v2 import SleeptimeMultiAgentV2
 from letta.helpers.datetime_helpers import get_utc_timestamp_ns
@@ -26,6 +26,7 @@ from letta.schemas.agent import AgentState, AgentType, CreateAgent, UpdateAgent
 from letta.schemas.agent_file import AgentFileSchema
 from letta.schemas.block import Block, BlockUpdate
 from letta.schemas.enums import JobType
+from letta.schemas.file import AgentFileAttachment, PaginatedAgentFiles
 from letta.schemas.group import Group
 from letta.schemas.job import JobStatus, JobUpdate, LettaRequestConfig
 from letta.schemas.letta_message import LettaMessageUnion, LettaMessageUpdateUnion, MessageType
@@ -39,6 +40,7 @@ from letta.schemas.source import Source
 from letta.schemas.tool import Tool
 from letta.schemas.user import User
 from letta.serialize_schemas.pydantic_agent_schema import AgentSchema
+from letta.server.rest_api.redis_stream_manager import create_background_stream_processor, redis_sse_stream_generator
 from letta.server.rest_api.utils import get_letta_server
 from letta.server.server import SyncServer
 from letta.services.summarizer.enums import SummarizationMode
@@ -249,6 +251,7 @@ async def import_agent(
     override_existing_tools: bool = True,
     project_id: str | None = None,
     strip_messages: bool = False,
+    env_vars: Optional[dict[str, Any]] = None,
 ) -> List[str]:
     """
     Import an agent using the new AgentFileSchema format.
@@ -259,7 +262,13 @@ async def import_agent(
         raise HTTPException(status_code=422, detail=f"Invalid agent file schema: {e!s}")
 
     try:
-        import_result = await server.agent_serialization_manager.import_file(schema=agent_schema, actor=actor)
+        import_result = await server.agent_serialization_manager.import_file(
+            schema=agent_schema,
+            actor=actor,
+            append_copy_suffix=append_copy_suffix,
+            override_existing_tools=override_existing_tools,
+            env_vars=env_vars,
+        )
 
         if not import_result.success:
             raise HTTPException(
@@ -297,7 +306,9 @@ async def import_agent_serialized(
         False,
         description="If set to True, strips all messages from the agent before importing.",
     ),
-    env_vars: Optional[Dict[str, Any]] = Form(None, description="Environment variables to pass to the agent for tool execution."),
+    env_vars_json: Optional[str] = Form(
+        None, description="Environment variables as a JSON string to pass to the agent for tool execution."
+    ),
 ):
     """
     Import a serialized agent file and recreate the agent(s) in the system.
@@ -311,6 +322,17 @@ async def import_agent_serialized(
     except json.JSONDecodeError:
         raise HTTPException(status_code=400, detail="Corrupted agent file format.")
 
+    # Parse env_vars_json if provided
+    env_vars = None
+    if env_vars_json:
+        try:
+            env_vars = json.loads(env_vars_json)
+        except json.JSONDecodeError:
+            raise HTTPException(status_code=400, detail="env_vars_json must be a valid JSON string")
+
+        if not isinstance(env_vars, dict):
+            raise HTTPException(status_code=400, detail="env_vars_json must be a valid JSON string")
+
     # Check if the JSON is AgentFileSchema or AgentSchema
     # TODO: This is kind of hacky, but should work as long as dont' change the schema
     if "agents" in agent_json and isinstance(agent_json.get("agents"), list):
@@ -323,6 +345,7 @@ async def import_agent_serialized(
             override_existing_tools=override_existing_tools,
             project_id=project_id,
             strip_messages=strip_messages,
+            env_vars=env_vars,
         )
     else:
         # This is a legacy AgentSchema
@@ -728,6 +751,49 @@ async def list_agent_folders(
     return await server.agent_manager.list_attached_sources_async(agent_id=agent_id, actor=actor)
 
 
+@router.get("/{agent_id}/files", response_model=PaginatedAgentFiles, operation_id="list_agent_files")
+async def list_agent_files(
+    agent_id: str,
+    cursor: Optional[str] = Query(None, description="Pagination cursor from previous response"),
+    limit: int = Query(20, ge=1, le=100, description="Number of items to return (1-100)"),
+    is_open: Optional[bool] = Query(None, description="Filter by open status (true for open files, false for closed files)"),
+    server: "SyncServer" = Depends(get_letta_server),
+    actor_id: str | None = Header(None, alias="user_id"),  # Extract user_id from header, default to None if not present
+):
+    """
+    Get the files attached to an agent with their open/closed status (paginated).
+    """
+    actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)
+
+    # get paginated file-agent relationships for this agent
+    file_agents, next_cursor, has_more = await server.file_agent_manager.list_files_for_agent_paginated(
+        agent_id=agent_id, actor=actor, cursor=cursor, limit=limit, is_open=is_open
+    )
+
+    # enrich with file and source metadata
+    enriched_files = []
+    for fa in file_agents:
+        # get source/folder metadata
+        source = await server.source_manager.get_source_by_id(source_id=fa.source_id, actor=actor)
+
+        # build response object
+        attachment = AgentFileAttachment(
+            id=fa.id,
+            file_id=fa.file_id,
+            file_name=fa.file_name,
+            folder_id=fa.source_id,
+            folder_name=source.name if source else "Unknown",
+            is_open=fa.is_open,
+            last_accessed_at=fa.last_accessed_at,
+            visible_content=fa.visible_content,
+            start_line=fa.start_line,
+            end_line=fa.end_line,
+        )
+        enriched_files.append(attachment)
+
+    return PaginatedAgentFiles(files=enriched_files, next_cursor=next_cursor, has_more=has_more)
+
+
 # TODO: remove? can also get with agent blocks
 @router.get("/{agent_id}/core-memory", response_model=Memory, operation_id="retrieve_agent_memory")
 async def retrieve_agent_memory(
@@ -999,7 +1065,8 @@ async def send_message(
         "bedrock",
         "ollama",
         "azure",
-        "together",
+        "xai",
+        "groq",
     ]
 
     # Create a new run for execution tracking
@@ -1143,7 +1210,8 @@ async def send_message_streaming(
         "bedrock",
         "ollama",
         "azure",
-        "together",
+        "xai",
+        "groq",
     ]
     model_compatible_token_streaming = agent.llm_config.model_endpoint_type in ["anthropic", "openai", "bedrock"]
 
@@ -1157,6 +1225,7 @@ async def send_message_streaming(
                 metadata={
                     "job_type": "send_message_streaming",
                     "agent_id": agent_id,
+                    "background": request.background or False,
                 },
                 request_config=LettaRequestConfig(
                     use_assistant_message=request.use_assistant_message,
@@ -1211,8 +1280,58 @@ async def send_message_streaming(
                         else SummarizationMode.PARTIAL_EVICT_MESSAGE_BUFFER
                     ),
                 )
+
             from letta.server.rest_api.streaming_response import StreamingResponseWithStatusCode, add_keepalive_to_stream
 
+            if request.background and settings.track_agent_run:
+                if isinstance(redis_client, NoopAsyncRedisClient):
+                    raise HTTPException(
+                        status_code=503,
+                        detail=(
+                            "Background streaming requires Redis to be running. "
+                            "Please ensure Redis is properly configured. "
+                            f"LETTA_REDIS_HOST: {settings.redis_host}, LETTA_REDIS_PORT: {settings.redis_port}"
+                        ),
+                    )
+
+                if request.stream_tokens and model_compatible_token_streaming:
+                    raw_stream = agent_loop.step_stream(
+                        input_messages=request.messages,
+                        max_steps=request.max_steps,
+                        use_assistant_message=request.use_assistant_message,
+                        request_start_timestamp_ns=request_start_timestamp_ns,
+                        include_return_message_types=request.include_return_message_types,
+                    )
+                else:
+                    raw_stream = agent_loop.step_stream_no_tokens(
+                        request.messages,
+                        max_steps=request.max_steps,
+                        use_assistant_message=request.use_assistant_message,
+                        request_start_timestamp_ns=request_start_timestamp_ns,
+                        include_return_message_types=request.include_return_message_types,
+                    )
+
+                asyncio.create_task(
+                    create_background_stream_processor(
+                        stream_generator=raw_stream,
+                        redis_client=redis_client,
+                        run_id=run.id,
+                    )
+                )
+
+                stream = redis_sse_stream_generator(
+                    redis_client=redis_client,
+                    run_id=run.id,
+                )
+
+                if request.include_pings and settings.enable_keepalive:
+                    stream = add_keepalive_to_stream(stream, keepalive_interval=settings.keepalive_interval)
+
+                return StreamingResponseWithStatusCode(
+                    stream,
+                    media_type="text/event-stream",
+                )
+
             if request.stream_tokens and model_compatible_token_streaming:
                 raw_stream = agent_loop.step_stream(
                     input_messages=request.messages,
@@ -1350,6 +1469,7 @@ async def _process_message_background(
             "google_vertex",
             "bedrock",
             "ollama",
+            "groq",
         ]
         if agent_eligible and model_compatible:
             if agent.enable_sleeptime and agent.agent_type != AgentType.voice_convo_agent:
@@ -1538,7 +1658,8 @@ async def preview_raw_payload(
         "bedrock",
         "ollama",
         "azure",
-        "together",
+        "xai",
+        "groq",
     ]
 
     if agent_eligible and model_compatible:
@@ -1608,7 +1729,8 @@ async def summarize_agent_conversation(
         "bedrock",
         "ollama",
         "azure",
-        "together",
+        "xai",
+        "groq",
     ]
 
     if agent_eligible and model_compatible:

letta/server/rest_api/routers/v1/folders.py

@@ -7,6 +7,7 @@ from typing import List, Optional
 
 from fastapi import APIRouter, Depends, Header, HTTPException, Query, UploadFile
 from starlette import status
+from starlette.responses import Response
 
 import letta.constants as constants
 from letta.helpers.pinecone_utils import (
@@ -34,7 +35,7 @@ from letta.services.file_processor.file_types import get_allowed_media_types, ge
 from letta.services.file_processor.parser.markitdown_parser import MarkitdownFileParser
 from letta.services.file_processor.parser.mistral_parser import MistralFileParser
 from letta.settings import settings
-from letta.utils import safe_create_task, sanitize_filename
+from letta.utils import safe_create_file_processing_task, safe_create_task, sanitize_filename
 
 logger = get_logger(__name__)
 
@@ -138,8 +139,11 @@ async def create_folder(
     # TODO: need to asyncify this
     if not folder_create.embedding_config:
         if not folder_create.embedding:
-            # TODO: modify error type
-            raise ValueError("Must specify either embedding or embedding_config in request")
+            if settings.default_embedding_handle is None:
+                # TODO: modify error type
+                raise ValueError("Must specify either embedding or embedding_config in request")
+            else:
+                folder_create.embedding = settings.default_embedding_handle
         folder_create.embedding_config = await server.get_embedding_config_from_handle_async(
             handle=folder_create.embedding,
             embedding_chunk_size=folder_create.embedding_chunk_size or constants.DEFAULT_EMBEDDING_CHUNK_SIZE,
@@ -257,13 +261,16 @@ async def upload_file_to_folder(
 
     # Store original filename and handle duplicate logic
     # Use custom name if provided, otherwise use the uploaded file's name
-    original_filename = sanitize_filename(name if name else file.filename)  # Basic sanitization only
+    # If custom name is provided, use it directly (it's just metadata, not a filesystem path)
+    # Otherwise, sanitize the uploaded filename for security
+    original_filename = name if name else sanitize_filename(file.filename)  # Basic sanitization only
 
     # Check if duplicate exists
     existing_file = await server.file_manager.get_file_by_original_name_and_source(
         original_filename=original_filename, source_id=folder_id, actor=actor
     )
 
+    unique_filename = None
     if existing_file:
         # Duplicate found, handle based on strategy
         if duplicate_handling == DuplicateFileHandling.ERROR:
@@ -305,8 +312,11 @@ async def upload_file_to_folder(
 
     # Use cloud processing for all files (simple files always, complex files with Mistral key)
     logger.info("Running experimental cloud based file processing...")
-    safe_create_task(
+    safe_create_file_processing_task(
         load_file_to_source_cloud(server, agent_states, content, folder_id, actor, folder.embedding_config, file_metadata),
+        file_metadata=file_metadata,
+        server=server,
+        actor=actor,
         logger=logger,
         label="file_processor.process",
     )