openrag 0.4.0.dev9__tar.gz → 0.4.1__tar.gz

{openrag-0.4.0.dev9/src/openrag.egg-info → openrag-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openrag
-Version: 0.4.0.dev9
+Version: 0.4.1
 Summary: OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables intelligent document search and AI-powered conversations.
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
@@ -15,7 +15,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Python: >=3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: agentd>=0.2.2
+Requires-Dist: agentd>=0.8.1
 Requires-Dist: aiofiles>=24.1.0
 Requires-Dist: cryptography>=45.0.6
 Requires-Dist: google-api-python-client>=2.143.0
@@ -37,6 +37,10 @@ Requires-Dist: python-dotenv>=1.0.0
 Requires-Dist: textual-fspicker>=0.6.0
 Requires-Dist: structlog>=25.4.0
 Requires-Dist: zxcvbn>=4.5.0
+Requires-Dist: litellm==1.83.3
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: tiktoken>=0.7.0
+Requires-Dist: openai>=2.30.0
 Dynamic: license-file
 
 <div align="center">

{openrag-0.4.0.dev9 → openrag-0.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "openrag"
-version = "0.4.0.dev9"
+version = "0.4.1"
 description = "OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables intelligent document search and AI-powered conversations."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -21,7 +21,7 @@ classifiers = [
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 dependencies = [
-    "agentd>=0.2.2",
+    "agentd>=0.8.1",
     "aiofiles>=24.1.0",
     "cryptography>=45.0.6",
     "google-api-python-client>=2.143.0",
@@ -42,7 +42,11 @@ dependencies = [
     "python-dotenv>=1.0.0",
     "textual-fspicker>=0.6.0",
     "structlog>=25.4.0",
-    "zxcvbn>=4.5.0"
+    "zxcvbn>=4.5.0",
+    "litellm==1.83.3",
+    "pyyaml>=6.0",
+    "tiktoken>=0.7.0",
+    "openai>=2.30.0",
 ]
 
 [dependency-groups]

{openrag-0.4.0.dev9 → openrag-0.4.1}/src/agent.py

@@ -158,7 +158,7 @@ async def async_response_stream(
                 else:
                     delta_text = str(chunk.delta)
                 full_response += delta_text
-            
+
             # Enhanced logging for tool call detection (Granite 3.3 8b investigation)
             chunk_attrs = dir(chunk) if hasattr(chunk, '__dict__') else []
             tool_related_attrs = [attr for attr in chunk_attrs if 'tool' in attr.lower() or 'call' in attr.lower() or 'retrieval' in attr.lower()]
@@ -180,7 +180,7 @@ async def async_response_stream(
                     chunk_data = chunk.__dict__
                 else:
                     chunk_data = str(chunk)
-                
+
                 # Log detailed chunk structure for investigation (especially for Granite 3.3 8b)
                 if isinstance(chunk_data, dict):
                     # Check for any fields that might indicate tool usage
@@ -218,7 +218,7 @@ async def async_response_stream(
                         'retrieved_documents' in chunk_data,
                         'retrieval_results' in chunk_data,
                     ])
-                    
+
                     if has_results:
                         logger.info(
                             "Detected implicit tool call in backend, injecting synthetic event",
@@ -242,7 +242,7 @@ async def async_response_stream(
                         # Send the synthetic event first
                         yield (json.dumps(synthetic_event, default=str) + "\n").encode("utf-8")
                         detected_tool_call = True  # Mark that we've injected a tool call
-                
+
                 yield (json.dumps(chunk_data, default=str) + "\n").encode("utf-8")
             except Exception as e:
                 # Fallback to string representation
@@ -626,20 +626,68 @@ async def async_langflow_chat(
 
     # Extract sources from retrieval tool calls in the response
     sources = []
+
+    # Layer 1: Structured output items (OpenAI Responses API format).
+    # Relaxed: check for any output item with a non-empty `results` field,
+    # regardless of `type` string (Langflow may use different type names).
     if hasattr(response_obj, "output") and response_obj.output:
         for output_item in response_obj.output:
-            item_type = getattr(output_item, "type", None)
-            if item_type in ("tool_call", "retrieval_call"):
-                for result in getattr(output_item, "results", None) or []:
-                    rd = result.model_dump() if hasattr(result, "model_dump") else (result if isinstance(result, dict) else {})
-                    if "text" in rd:
-                        sources.append({
-                            "filename": rd.get("filename", ""),
-                            "text": rd.get("text", ""),
-                            "score": rd.get("score", 0),
-                            "page": rd.get("page"),
-                            "mimetype": rd.get("mimetype"),
-                        })
+            for result in getattr(output_item, "results", None) or []:
+                rd = (
+                    result.model_dump()
+                    if hasattr(result, "model_dump")
+                    else (result if isinstance(result, dict) else {})
+                )
+                if "text" in rd:
+                    sources.append({
+                        "filename": rd.get("filename", ""),
+                        "text": rd.get("text", ""),
+                        "score": rd.get("score", 0),
+                        "page": rd.get("page"),
+                        "mimetype": rd.get("mimetype"),
+                    })
+
+    # Layer 2: Top-level dict inspection (mirrors streaming middleware in async_response_stream).
+    # Langflow may embed retrieval results directly in the response dict rather than
+    # inside typed output items.
+    if not sources:
+        resp_dict = (
+            response_obj.model_dump()
+            if hasattr(response_obj, "model_dump")
+            else getattr(response_obj, "__dict__", {})
+        )
+        implicit_results = (
+            resp_dict.get("results")
+            or resp_dict.get("outputs")
+            or resp_dict.get("retrieved_documents")
+            or resp_dict.get("retrieval_results")
+            or []
+        )
+        if isinstance(implicit_results, list):
+            for result in implicit_results:
+                if isinstance(result, dict) and "text" in result:
+                    sources.append({
+                        "filename": result.get("filename", ""),
+                        "text": result.get("text", ""),
+                        "score": result.get("score", 0),
+                        "page": result.get("page"),
+                        "mimetype": result.get("mimetype"),
+                    })
+
+    # Layer 3: Citation-text fallback.
+    # Parse "(Source: filename)" patterns emitted by the LLM when it cites documents.
+    # This is the last-resort fallback when Langflow's response object carries no
+    # structured retrieval data.
+    if not sources:
+        import re
+        for match in re.finditer(r"\(Source:\s*([^\)]+)\)", response_text):
+            sources.append({
+                "filename": match.group(1).strip(),
+                "text": "",
+                "score": 0,
+                "page": None,
+                "mimetype": None,
+            })
 
     if not store_conversation:
         return response_text, response_id, sources
@@ -739,7 +787,7 @@ async def async_langflow_chat_stream(
                     response_id = chunk_data["id"]
                 elif "response_id" in chunk_data:
                     response_id = chunk_data["response_id"]
-                
+
                 # Check for error status
                 if chunk_data.get("finish_reason") == "error" or chunk_data.get("status") == "failed":
                     error_occurred = True
@@ -788,7 +836,7 @@ async def async_langflow_chat_stream(
         # Log the error
         logger.error(f"Error in langflow chat stream: {e}", exc_info=True)
         error_occurred = True
-        
+
         # Store error message in conversation history so it persists
         error_message = {
             "role": "assistant",
@@ -797,19 +845,19 @@ async def async_langflow_chat_stream(
             "error": True,
         }
         conversation_state["messages"].append(error_message)
-        
+
         # Try to store the conversation with error message
         # Use a temporary response_id if we don't have one
         if not response_id:
             response_id = f"error_{user_id}_{int(datetime.now().timestamp())}"
-        
+
         try:
             conversation_state["last_activity"] = datetime.now()
             await store_conversation_thread(user_id, response_id, conversation_state)
             logger.debug(f"Stored conversation with error for user {user_id}")
         except Exception as store_error:
             logger.error(f"Failed to store error conversation: {store_error}")
-        
+
         # Re-raise the exception so it propagates to the API layer
         raise
 

{openrag-0.4.0.dev9 → openrag-0.4.1}/src/api/settings.py

@@ -1,3 +1,4 @@
+from dependencies import get_models_service
 import asyncio
 import json
 import platform
@@ -9,7 +10,6 @@ from utils.telemetry import TelemetryClient, Category, MessageId
 from utils.version_utils import OPENRAG_VERSION
 from config.settings import (
     DEFAULT_DOCS_URL,
-    DISABLE_INGEST_WITH_LANGFLOW,
     INGEST_SAMPLE_DATA,
     LANGFLOW_URL,
     LANGFLOW_CHAT_FLOW_ID,
@@ -885,6 +885,7 @@ async def onboarding(
     flows_service=Depends(get_flows_service),
     session_manager=Depends(get_session_manager),
     document_service=Depends(get_document_service),
+    models_service=Depends(get_models_service),
     task_service=Depends(get_task_service),
     langflow_file_service=Depends(get_langflow_file_service),
     knowledge_filter_service=Depends(get_knowledge_filter_service),
@@ -936,7 +937,7 @@ async def onboarding(
         embedding_model_selected = None
         embedding_provider_selected = None
 
-        if body.embedding_model and not DISABLE_INGEST_WITH_LANGFLOW:
+        if body.embedding_model:
             embedding_model_selected = body.embedding_model.strip()
             current_config.knowledge.embedding_model = embedding_model_selected
             config_updated = True
@@ -1151,8 +1152,12 @@ async def onboarding(
                     # Import the function here to avoid circular imports
                     from main import ingest_default_documents_when_ready
 
+                    if not config_manager.save_config_file(current_config):
+                        logger.error("Failed to save embedding model to config")
+
                     task_id = await ingest_default_documents_when_ready(
                         document_service,
+                        models_service,
                         task_service,
                         langflow_file_service,
                         session_manager,
@@ -1664,8 +1669,6 @@ async def rollback_onboarding(
                 {"error": "No onboarding configuration to rollback"}, status_code=400
             )
 
-            jwt_token = user.jwt_token
-
         logger.info("Rolling back onboarding configuration due to file failures")
 
         # Get all tasks for the user
@@ -1728,7 +1731,7 @@ async def rollback_onboarding(
                         if filename:
                             try:
                                 opensearch_client = session_manager.get_user_opensearch_client(
-                                    user.user_id, jwt_token
+                                    user.user_id, user.jwt_token
                                 )
                                 from utils.opensearch_queries import build_filename_delete_body
                                 from config.settings import get_index_name
@@ -1890,6 +1893,7 @@ async def update_docling_preset(
 async def refresh_openrag_docs(
     document_service=Depends(get_document_service),
     task_service=Depends(get_task_service),
+    models_service=Depends(get_models_service),
     langflow_file_service=Depends(get_langflow_file_service),
     session_manager=Depends(get_session_manager),
     user: User = Depends(get_current_user),
@@ -1900,6 +1904,7 @@ async def refresh_openrag_docs(
 
         refreshed = await refresh_default_openrag_docs(
             document_service=document_service,
+            models_service=models_service,
             task_service=task_service,
             langflow_file_service=langflow_file_service,
             session_manager=session_manager,

{openrag-0.4.0.dev9 → openrag-0.4.1}/src/api/upload.py

@@ -1,3 +1,4 @@
+from dependencies import get_models_service
 import os
 from typing import Optional
 from urllib.parse import urlparse
@@ -35,15 +36,10 @@ async def upload(
     try:
 
         from config.settings import is_no_auth_mode
-
-        if is_no_auth_mode():
-            owner_user_id = None
-            owner_name = None
-            owner_email = None
-        else:
-            owner_user_id = user.user_id
-            owner_name = user.name
-            owner_email = user.email
+        is_no_auth = is_no_auth_mode()
+        owner_user_id = user.user_id if (user and not is_no_auth) else None
+        owner_name = user.name if user else None
+        owner_email = user.email if user else None
 
         result = await document_service.process_upload_file(
             file,
@@ -84,15 +80,10 @@ async def upload_path(
     jwt_token = user.jwt_token
 
     from config.settings import is_no_auth_mode
-
-    if is_no_auth_mode():
-        owner_user_id = None
-        owner_name = None
-        owner_email = None
-    else:
-        owner_user_id = user.user_id
-        owner_name = user.name
-        owner_email = user.email
+    is_no_auth = is_no_auth_mode()
+    owner_user_id = user.user_id if (user and not is_no_auth) else None
+    owner_name = user.name if user else None
+    owner_email = user.email if user else None
 
     from api.documents import _ensure_index_exists
     await _ensure_index_exists()
@@ -163,6 +154,7 @@ async def upload_options(
 async def upload_bucket(
     body: UploadBucketBody,
     task_service=Depends(get_task_service),
+    models_service=Depends(get_models_service),
     session_manager=Depends(get_session_manager),
     user: User = Depends(get_current_user),
 ):
@@ -194,18 +186,13 @@ async def upload_bucket(
     jwt_token = user.jwt_token
 
     from models.processors import S3FileProcessor
-    from config.settings import is_no_auth_mode
 
-    if is_no_auth_mode():
-        owner_user_id = None
-        owner_name = None
-        owner_email = None
-        task_user_id = None
-    else:
-        owner_user_id = user.user_id
-        owner_name = user.name
-        owner_email = user.email
-        task_user_id = user.user_id
+    from config.settings import is_no_auth_mode
+    is_no_auth = is_no_auth_mode()
+    owner_user_id = user.user_id if (user and not is_no_auth) else None
+    owner_name = user.name if user else None
+    owner_email = user.email if user else None
+    task_user_id = user.user_id if (user and not is_no_auth) else None
 
     from api.documents import _ensure_index_exists
     await _ensure_index_exists()
@@ -213,6 +200,7 @@ async def upload_bucket(
     processor = S3FileProcessor(
         task_service.document_service,
         bucket,
+        models_service=models_service,
         s3_client=s3_client,
         owner_user_id=owner_user_id,
         jwt_token=jwt_token,

openrag-0.4.1/src/config/embedding_constants.py

@@ -0,0 +1,4 @@
+"""Embedding model constants."""
+
+OPENAI_DEFAULT_EMBEDDING_MODEL = "text-embedding-3-small"
+OPENAI_EMBEDDING_MODEL_PREFIX = "text-embedding"

{openrag-0.4.0.dev9 → openrag-0.4.1}/src/config/model_constants.py

@@ -31,8 +31,6 @@ OPENAI_VALIDATION_MODELS = [
 ]
 
 OPENAI_DEFAULT_LANGUAGE_MODEL = "gpt-4o"
-OPENAI_DEFAULT_EMBEDDING_MODEL = "text-embedding-3-small"
-OPENAI_EMBEDDING_MODEL_PREFIX = "text-embedding"
 
 ANTHROPIC_DEFAULT_LANGUAGE_MODEL = "claude-sonnet-4-5-20250929"
 

{openrag-0.4.0.dev9 → openrag-0.4.1}/src/config/settings.py

@@ -1,5 +1,7 @@
 import asyncio
 import os
+import threading
+import concurrent.futures
 from utils.env_utils import get_env_int, get_env_float
 
 import httpx
@@ -8,6 +10,7 @@ from dotenv import load_dotenv
 from openai import AsyncOpenAI
 from opensearchpy import AsyncOpenSearch
 from opensearchpy._async.http_aiohttp import AIOHttpConnection
+from config.embedding_constants import OPENAI_DEFAULT_EMBEDDING_MODEL
 
 from utils.container_utils import get_container_host
 from utils.logging_config import get_logger
@@ -107,26 +110,6 @@ WEBHOOK_BASE_URL = os.getenv(
 VECTOR_DIM = 1536
 KNN_EF_CONSTRUCTION = 100
 KNN_M = 16
-EMBED_MODEL = "text-embedding-3-small"
-
-OPENAI_EMBEDDING_DIMENSIONS = {
-        "text-embedding-3-small": 1536,
-        "text-embedding-3-large": 3072,
-        "text-embedding-ada-002": 1536,
-    }
-
-WATSONX_EMBEDDING_DIMENSIONS = {
-# IBM Models
-"ibm/granite-embedding-107m-multilingual": 384,
-"ibm/granite-embedding-278m-multilingual": 1024,
-"ibm/slate-125m-english-rtrvr": 768,
-"ibm/slate-125m-english-rtrvr-v2": 768,
-"ibm/slate-30m-english-rtrvr": 384,
-"ibm/slate-30m-english-rtrvr-v2": 384,
-# Third Party Models
-"intfloat/multilingual-e5-large": 1024,
-"sentence-transformers/all-minilm-l6-v2": 384,
-}
 
 INDEX_BODY = {
     "settings": {
@@ -320,7 +303,7 @@ class AppClients:
         self.langflow_client = None
         self.langflow_http_client = None
         self._patched_async_client = None  # Private attribute - single client for all providers
-        self._client_init_lock = __import__('threading').Lock()  # Lock for thread-safe initialization
+        self._client_init_lock = threading.Lock()  # Lock for thread-safe initialization
         self.docling_http_client = None
 
     async def initialize(self):
@@ -455,6 +438,11 @@ class AppClients:
                 if config.providers.openai.api_key:
                     os.environ["OPENAI_API_KEY"] = config.providers.openai.api_key
                     logger.debug("Loaded OpenAI API key from config")
+                elif not os.environ.get("OPENAI_API_KEY"):
+                    # Provide dummy key to satisfy AsyncOpenAI constructor; 
+                    # LiteLLM/MCP will handle routing to other providers if needed.
+                    os.environ["OPENAI_API_KEY"] = "no-key-required"
+                    logger.debug("Using dummy OpenAI API key to satisfy client constructor")
 
                 # Set Anthropic credentials
                 if config.providers.anthropic.api_key:
@@ -478,14 +466,23 @@ class AppClients:
                     os.environ["OLLAMA_ENDPOINT"] = config.providers.ollama.endpoint
                     logger.debug("Loaded Ollama endpoint from config")
 
+                # Determine model and provider for both probe and production client
+                model_name = config.knowledge.embedding_model or OPENAI_DEFAULT_EMBEDDING_MODEL
+                provider = config.knowledge.embedding_provider or "openai"
             except Exception as e:
                 logger.debug("Could not load provider credentials from config", error=str(e))
+                # Provide fallbacks if config loading failed
+                model_name = OPENAI_DEFAULT_EMBEDDING_MODEL
+                provider = "openai"
+                # Ensure a dummy key is available to satisfy the AsyncOpenAI constructor
+                # and avoid AuthenticationError if config loading failed.
+                if not os.environ.get("OPENAI_API_KEY"):
+                    os.environ["OPENAI_API_KEY"] = "no-key-required"
+                    logger.debug("Using dummy OpenAI API key fallback (config load failed)")
+
 
-            # Try to initialize the client - AsyncOpenAI() will read from environment
-            # We'll try HTTP/2 first with a probe, then fall back to HTTP/1.1 if it times out
-            import asyncio
-            import concurrent.futures
-            import threading
+            # API key for AsyncOpenAI constructor
+            api_key = os.environ.get("OPENAI_API_KEY")
 
             async def probe_http2():
                 """Returns True if HTTP/2 works, False to fall back to HTTP/1.1.
@@ -495,20 +492,21 @@ class AppClients:
                 production client is created after this thread exits, in the
                 caller's event loop, avoiding cross-loop SSL transport errors.
                 """
-                client = AsyncOpenAI()
-                logger.info("Probing OpenAI client with HTTP/2...")
+                # Use a standard OpenAI client for the probe (only runs for OpenAI provider)
+                client = AsyncOpenAI(api_key=api_key)
+                logger.info(f"Probing client with HTTP/2 using model {model_name}...")
                 try:
                     await asyncio.wait_for(
                         client.embeddings.create(
-                            model='text-embedding-3-small',
+                            model=model_name,
                             input=['test']
                         ),
                         timeout=5.0
                     )
-                    logger.info("HTTP/2 probe successful")
+                    logger.info(f"HTTP/2 probe successful with {model_name}")
                     return True
                 except (asyncio.TimeoutError, Exception) as probe_error:
-                    logger.warning("HTTP/2 probe failed, falling back to HTTP/1.1", error=str(probe_error))
+                    logger.warning(f"HTTP/2 probe failed with {model_name}, falling back to HTTP/1.1", error=str(probe_error))
                     return False
                 finally:
                     # Always close the probe client so its connections are fully
@@ -528,26 +526,32 @@ class AppClients:
                     loop.close()
 
             try:
-                # Run the probe in a separate thread with its own event loop.
-                # Only the probe result (bool) crosses the thread boundary;
-                # the production client is created here so its connections are
-                # bound to the caller's event loop, not the (now closed) probe loop.
-                with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
-                    future = executor.submit(run_probe_in_thread)
-                    use_http2 = future.result(timeout=15)
+                # Run the probe only for OpenAI provider; local and other providers
+                # (Ollama, WatsonX) typically use HTTP/1.1 for reliability.
+                if provider.lower() == "openai":
+                    # Run the probe in a separate thread with its own event loop.
+                    # Only the probe result (bool) crosses the thread boundary;
+                    # the production client is created here so its connections are
+                    # bound to the caller's event loop, not the (now closed) probe loop.
+                    with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
+                        future = executor.submit(run_probe_in_thread)
+                        use_http2 = future.result(timeout=15)
+                else:
+                    use_http2 = False
+                    logger.debug(f"Skipping HTTP/2 probe for provider: {provider}")
 
                 if use_http2:
-                    self._patched_async_client = patch_openai_with_mcp(AsyncOpenAI())
-                    logger.info("OpenAI client initialized with HTTP/2")
+                    self._patched_async_client = patch_openai_with_mcp(AsyncOpenAI(api_key=api_key))
+                    logger.info(f"OpenAI-compatible client initialized with HTTP/2 (model: {model_name})")
                 else:
                     http_client = httpx.AsyncClient(
                         http2=False,
                         timeout=httpx.Timeout(60.0, connect=10.0)
                     )
                     self._patched_async_client = patch_openai_with_mcp(
-                        AsyncOpenAI(http_client=http_client)
+                        AsyncOpenAI(api_key=api_key, http_client=http_client)
                     )
-                    logger.info("OpenAI client initialized with HTTP/1.1 (fallback)")
+                    logger.info(f"OpenAI-compatible client initialized with HTTP/1.1 fallback (model: {model_name})")
                 logger.info("Successfully initialized OpenAI client")
             except Exception as e:
                 logger.error(f"Failed to initialize OpenAI client: {e.__class__.__name__}: {str(e)}")
@@ -864,7 +868,7 @@ def get_openrag_config():
 # Expose configuration settings for backward compatibility and easy access
 def get_provider_config():
     """Get provider configuration."""
-    return get_openrag_config().provider
+    return get_openrag_config().providers
 
 
 def get_knowledge_config():
@@ -879,7 +883,7 @@ def get_agent_config():
 
 def get_embedding_model() -> str:
     """Return the currently configured embedding model."""
-    return get_openrag_config().knowledge.embedding_model or EMBED_MODEL if DISABLE_INGEST_WITH_LANGFLOW else ""
+    return get_openrag_config().knowledge.embedding_model or (OPENAI_DEFAULT_EMBEDDING_MODEL if DISABLE_INGEST_WITH_LANGFLOW else "")
 
 
 def get_index_name() -> str:

{openrag-0.4.0.dev9 → openrag-0.4.1}/src/connectors/service.py

@@ -20,6 +20,8 @@ class ConnectorService:
         index_name: str,
         task_service=None,
         session_manager=None,
+        models_service=None,
+        document_service=None,
     ):
         self.clients = patched_async_client
         self.embed_model = embed_model
@@ -27,6 +29,8 @@ class ConnectorService:
         self.task_service = task_service
         self.session_manager = session_manager
         self.connection_manager = ConnectionManager()
+        self.models_service = models_service
+        self.document_service = document_service
 
     async def initialize(self):
         """Initialize the service by loading existing connections"""
@@ -57,18 +61,12 @@ class ConnectorService:
             with open(tmp_path, "wb") as f:
                 f.write(document.content)
 
-            # Use existing process_file_common function with connector document metadata
-            # We'll use the document service's process_file_common method
-            from services.document_service import DocumentService
-
-            doc_service = DocumentService(session_manager=self.session_manager)
-
             logger.debug("Processing connector document", document_id=document.id)
 
             # Process using consolidated processing pipeline
             from models.processors import TaskProcessor
 
-            processor = TaskProcessor(document_service=doc_service)
+            processor = TaskProcessor(document_service=self.document_service, models_service=self.models_service)
             result = await processor.process_document_standard(
                 file_path=tmp_path,
                 file_hash=document.id,  # Use connector document ID as hash
@@ -284,6 +282,7 @@ class ConnectorService:
                 if self.task_service and self.task_service.document_service
                 else DocumentService(session_manager=self.session_manager)
             ),
+            models_service=self.models_service,
         )
 
         # Use file IDs as items (no more fake file paths!)
@@ -415,6 +414,7 @@ class ConnectorService:
                 if self.task_service and self.task_service.document_service
                 else DocumentService(session_manager=self.session_manager)
             ),
+            models_service=self.models_service,
         )
 
         # Create custom task using TaskService