openrag 0.3.2.dev31__tar.gz → 0.3.2.dev41__tar.gz

{openrag-0.3.2.dev31/src/openrag.egg-info → openrag-0.3.2.dev41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openrag
-Version: 0.3.2.dev31
+Version: 0.3.2.dev41
 Summary: OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables intelligent document search and AI-powered conversations.
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
@@ -29,6 +29,7 @@ Requires-Dist: python-multipart>=0.0.20
 Requires-Dist: fastapi>=0.115.0
 Requires-Dist: uvicorn>=0.35.0
 Requires-Dist: boto3>=1.35.0
+Requires-Dist: ibm-cos-sdk>=2.13.0
 Requires-Dist: psutil>=7.0.0
 Requires-Dist: rich>=13.0.0
 Requires-Dist: textual>=0.45.0

{openrag-0.3.2.dev31 → openrag-0.3.2.dev41}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "openrag"
-version = "0.3.2.dev31"
+version = "0.3.2.dev41"
 description = "OpenRAG is a comprehensive Retrieval-Augmented Generation platform that enables intelligent document search and AI-powered conversations."
 readme = "README.md"
 requires-python = ">=3.13"
@@ -35,6 +35,7 @@ dependencies = [
     "fastapi>=0.115.0",
     "uvicorn>=0.35.0",
     "boto3>=1.35.0",
+    "ibm-cos-sdk>=2.13.0",
     "psutil>=7.0.0",
     "rich>=13.0.0",
     "textual>=0.45.0",

{openrag-0.3.2.dev31 → openrag-0.3.2.dev41}/src/api/connectors.py

@@ -88,6 +88,11 @@ async def get_synced_file_ids_for_connector(
 class ConnectorSyncBody(BaseModel):
     max_files: Optional[int] = None
     selected_files: Optional[List[Any]] = None
+    # When True, ingest ALL files from the connector (bypasses the existing-files gate).
+    # Used by direct-sync providers like IBM COS on initial ingest.
+    sync_all: bool = False
+    # When set, only ingest files from these buckets (IBM COS specific).
+    bucket_filter: Optional[List[str]] = None
 
 
 async def list_connectors(
@@ -96,8 +101,8 @@ async def list_connectors(
 ):
     """List available connector types with metadata"""
     try:
-        connector_types = (
-            connector_service.connection_manager.get_available_connector_types()
+        connector_types = connector_service.connection_manager.get_available_connector_types(
+            user_id=user.user_id
         )
         return JSONResponse({"connectors": connector_types})
     except Exception as e:
@@ -200,6 +205,51 @@ async def connector_sync(
                 jwt_token=jwt_token,
                 file_infos=file_infos,
             )
+        elif body.sync_all or body.bucket_filter:
+            # Full ingest: discover and ingest all files (or files from specific buckets).
+            # Used by direct-sync providers (IBM COS) on initial ingest or per-bucket sync.
+            logger.info(
+                "Full connector ingest requested",
+                connector_type=connector_type,
+                bucket_filter=body.bucket_filter,
+            )
+            connector = await connector_service.get_connector(working_connection.connection_id)
+            if body.bucket_filter:
+                # List only files from the requested buckets, then sync_specific_files
+                original_buckets = connector.bucket_names
+                connector.bucket_names = body.bucket_filter
+                try:
+                    all_file_ids = []
+                    page_token = None
+                    while True:
+                        result = await connector.list_files(page_token=page_token)
+                        for f in result.get("files", []):
+                            all_file_ids.append(f["id"])
+                        page_token = result.get("next_page_token")
+                        if not page_token:
+                            break
+                finally:
+                    connector.bucket_names = original_buckets
+
+                if not all_file_ids:
+                    return JSONResponse(
+                        {"status": "no_files", "message": "No files found in the selected buckets."},
+                        status_code=200,
+                    )
+                task_id = await connector_service.sync_specific_files(
+                    working_connection.connection_id,
+                    user.user_id,
+                    all_file_ids,
+                    jwt_token=jwt_token,
+                )
+            else:
+                # sync_all: ingest everything the connector can see
+                task_id = await connector_service.sync_connector_files(
+                    working_connection.connection_id,
+                    user.user_id,
+                    max_files=max_files,
+                    jwt_token=jwt_token,
+                )
         else:
             # No files specified - sync only files already in OpenSearch for this connector
             # This ensures deleted files stay deleted
@@ -209,7 +259,7 @@ async def connector_sync(
                 session_manager=session_manager,
                 jwt_token=jwt_token,
             )
-            
+
             if not existing_file_ids and not existing_filenames:
                 return JSONResponse(
                     {
@@ -218,7 +268,7 @@ async def connector_sync(
                     },
                     status_code=200,
                 )
-            
+
             # If we have document_ids (connector file IDs), use sync_specific_files
             # Otherwise, use filename filtering with sync_connector_files
             if existing_file_ids:
@@ -602,6 +652,8 @@ async def connector_disconnect(
         )
 
 
+# ---------------------------------------------------------------------------
+
 async def sync_all_connectors(
     connector_service=Depends(get_connector_service),
     session_manager=Depends(get_session_manager),
@@ -615,7 +667,7 @@ async def sync_all_connectors(
         jwt_token = user.jwt_token
 
         # Cloud connector types to sync
-        cloud_connector_types = ["google_drive", "onedrive", "sharepoint"]
+        cloud_connector_types = ["google_drive", "onedrive", "sharepoint", "ibm_cos", "aws_s3"]
         
         all_task_ids = []
         synced_connectors = []

{openrag-0.3.2.dev31 → openrag-0.3.2.dev41}/src/config/settings.py

@@ -51,6 +51,8 @@ GOOGLE_OAUTH_CLIENT_ID = os.getenv("GOOGLE_OAUTH_CLIENT_ID")
 GOOGLE_OAUTH_CLIENT_SECRET = os.getenv("GOOGLE_OAUTH_CLIENT_SECRET")
 DOCLING_OCR_ENGINE = os.getenv("DOCLING_OCR_ENGINE")
 
+IBM_AUTH_ENABLED = os.getenv("IBM_AUTH_ENABLED", "false").lower() in ("true", "1", "yes")
+
 # Ingestion configuration
 DISABLE_INGEST_WITH_LANGFLOW = os.getenv(
     "DISABLE_INGEST_WITH_LANGFLOW", "false"

{openrag-0.3.2.dev31 → openrag-0.3.2.dev41}/src/connectors/__init__.py

@@ -2,10 +2,14 @@ from .base import BaseConnector
 from .google_drive import GoogleDriveConnector
 from .sharepoint import SharePointConnector
 from .onedrive import OneDriveConnector
+from .ibm_cos import IBMCOSConnector
+from .aws_s3 import S3Connector
 
 __all__ = [
     "BaseConnector",
     "GoogleDriveConnector",
     "SharePointConnector",
     "OneDriveConnector",
+    "IBMCOSConnector",
+    "S3Connector",
 ]

openrag-0.3.2.dev41/src/connectors/aws_s3/__init__.py

@@ -0,0 +1,19 @@
+"""Amazon S3 / S3-compatible connector for OpenRAG."""
+
+from .connector import S3Connector
+from .models import S3ConfigureBody
+from .api import (
+    s3_defaults,
+    s3_configure,
+    s3_list_buckets,
+    s3_bucket_status,
+)
+
+__all__ = [
+    "S3Connector",
+    "S3ConfigureBody",
+    "s3_defaults",
+    "s3_configure",
+    "s3_list_buckets",
+    "s3_bucket_status",
+]

openrag-0.3.2.dev41/src/connectors/aws_s3/api.py

@@ -0,0 +1,175 @@
+"""FastAPI route handlers for AWS S3-specific endpoints."""
+
+import os
+
+from fastapi import Depends
+from fastapi.responses import JSONResponse
+
+from config.settings import get_index_name
+from dependencies import get_connector_service, get_session_manager, get_current_user
+from session_manager import User
+from utils.logging_config import get_logger
+
+from .auth import create_s3_resource
+from .models import S3ConfigureBody
+from .support import build_s3_config
+
+logger = get_logger(__name__)
+
+
+async def s3_defaults(
+    connector_service=Depends(get_connector_service),
+    user: User = Depends(get_current_user),
+):
+    """Return current S3 env-var defaults for pre-filling the config dialog.
+
+    Sensitive values (secret key) are masked — only whether they are set is returned.
+    """
+    access_key = os.getenv("AWS_ACCESS_KEY_ID", "")
+    secret_key = os.getenv("AWS_SECRET_ACCESS_KEY", "")
+    endpoint_url = os.getenv("AWS_S3_ENDPOINT", "")
+    region = os.getenv("AWS_REGION", "")
+
+    connections = await connector_service.connection_manager.list_connections(
+        user_id=user.user_id, connector_type="aws_s3"
+    )
+    conn_config = connections[0].config or {} if connections else {}
+
+    def _pick(conn_key, env_val):
+        return conn_config.get(conn_key) or env_val
+
+    return JSONResponse({
+        "access_key_set": bool(access_key or conn_config.get("access_key")),
+        "secret_key_set": bool(secret_key or conn_config.get("secret_key")),
+        "endpoint": _pick("endpoint_url", endpoint_url),
+        "region": _pick("region", region),
+        "bucket_names": conn_config.get("bucket_names", []),
+        "connection_id": connections[0].connection_id if connections else None,
+    })
+
+
+async def s3_configure(
+    body: S3ConfigureBody,
+    connector_service=Depends(get_connector_service),
+    user: User = Depends(get_current_user),
+):
+    """Create or update an S3 connection with explicit credentials.
+
+    Tests the credentials by listing buckets, then persists the connection.
+    """
+    existing_connections = await connector_service.connection_manager.list_connections(
+        user_id=user.user_id, connector_type="aws_s3"
+    )
+    existing_config = existing_connections[0].config if existing_connections else {}
+
+    conn_config, error = build_s3_config(body, existing_config)
+    if error:
+        return JSONResponse({"error": error}, status_code=400)
+
+    # Test credentials
+    try:
+        s3 = create_s3_resource(conn_config)
+        list(s3.buckets.all())
+    except Exception:
+        logger.exception("Failed to connect to S3 during credential test.")
+        return JSONResponse(
+            {"error": "Could not connect to S3 with the provided configuration."},
+            status_code=400,
+        )
+
+    # Persist: update existing connection or create a new one
+    if body.connection_id:
+        existing = await connector_service.connection_manager.get_connection(body.connection_id)
+        if existing and existing.user_id == user.user_id:
+            await connector_service.connection_manager.update_connection(
+                connection_id=body.connection_id,
+                config=conn_config,
+            )
+            connector_service.connection_manager.active_connectors.pop(body.connection_id, None)
+            return JSONResponse({"connection_id": body.connection_id, "status": "connected"})
+
+    connection_id = await connector_service.connection_manager.create_connection(
+        connector_type="aws_s3",
+        name="Amazon S3",
+        config=conn_config,
+        user_id=user.user_id,
+    )
+    return JSONResponse({"connection_id": connection_id, "status": "connected"})
+
+
+async def s3_list_buckets(
+    connection_id: str,
+    connector_service=Depends(get_connector_service),
+    user: User = Depends(get_current_user),
+):
+    """List all buckets accessible with the stored S3 credentials."""
+    connection = await connector_service.connection_manager.get_connection(connection_id)
+    if not connection or connection.user_id != user.user_id:
+        return JSONResponse({"error": "Connection not found"}, status_code=404)
+    if connection.connector_type != "aws_s3":
+        return JSONResponse({"error": "Not an S3 connection"}, status_code=400)
+
+    try:
+        s3 = create_s3_resource(connection.config)
+        buckets = [b.name for b in s3.buckets.all()]
+        return JSONResponse({"buckets": buckets})
+    except Exception:
+        logger.exception("Failed to list S3 buckets for connection %s", connection_id)
+        return JSONResponse({"error": "Failed to list buckets"}, status_code=500)
+
+
+async def s3_bucket_status(
+    connection_id: str,
+    connector_service=Depends(get_connector_service),
+    session_manager=Depends(get_session_manager),
+    user: User = Depends(get_current_user),
+):
+    """Return all buckets for an S3 connection with their ingestion status."""
+    connection = await connector_service.connection_manager.get_connection(connection_id)
+    if not connection or connection.user_id != user.user_id:
+        return JSONResponse({"error": "Connection not found"}, status_code=404)
+    if connection.connector_type != "aws_s3":
+        return JSONResponse({"error": "Not an S3 connection"}, status_code=400)
+
+    # 1. List all buckets from S3
+    try:
+        s3 = create_s3_resource(connection.config)
+        all_buckets = [b.name for b in s3.buckets.all()]
+    except Exception as exc:
+        logger.exception("Failed to list buckets from S3 for connection %s", connection_id)
+        return JSONResponse({"error": "Failed to list buckets"}, status_code=500)
+
+    # 2. Count indexed documents per bucket from OpenSearch
+    ingested_counts: dict = {}
+    try:
+        opensearch_client = session_manager.get_user_opensearch_client(
+            user.user_id, user.jwt_token
+        )
+        query_body = {
+            "size": 0,
+            "query": {"term": {"connector_type": "aws_s3"}},
+            "aggs": {
+                "doc_ids": {
+                    "terms": {"field": "document_id", "size": 50000}
+                }
+            },
+        }
+        index_name = get_index_name()
+        os_resp = opensearch_client.search(index=index_name, body=query_body)
+        for bucket_entry in os_resp.get("aggregations", {}).get("doc_ids", {}).get("buckets", []):
+            doc_id = bucket_entry["key"]
+            if "::" in doc_id:
+                bucket_name = doc_id.split("::")[0]
+                ingested_counts[bucket_name] = ingested_counts.get(bucket_name, 0) + 1
+    except Exception:
+        pass  # OpenSearch unavailable — show zero counts
+
+    result = [
+        {
+            "name": bucket,
+            "ingested_count": ingested_counts.get(bucket, 0),
+            "is_synced": ingested_counts.get(bucket, 0) > 0,
+        }
+        for bucket in all_buckets
+    ]
+    return JSONResponse({"buckets": result})

openrag-0.3.2.dev41/src/connectors/aws_s3/auth.py

@@ -0,0 +1,90 @@
+"""Amazon S3 / S3-compatible storage authentication and client factory."""
+
+import os
+from typing import Any, Dict, Optional
+
+from utils.logging_config import get_logger
+
+logger = get_logger(__name__)
+
+_DEFAULT_REGION = "us-east-1"
+
+
+def _resolve_credentials(config: Dict[str, Any]) -> Dict[str, Any]:
+    """Resolve S3 credentials from config dict with environment variable fallback.
+
+    Resolution order for each value: config dict → environment variable → default.
+
+    Raises:
+        ValueError: If access_key or secret_key cannot be resolved.
+    """
+    access_key: Optional[str] = config.get("access_key") or os.getenv("AWS_ACCESS_KEY_ID")
+    secret_key: Optional[str] = config.get("secret_key") or os.getenv("AWS_SECRET_ACCESS_KEY")
+
+    if not access_key or not secret_key:
+        raise ValueError(
+            "S3 credentials are required. Provide 'access_key' and 'secret_key' in the "
+            "connector config, or set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY env vars."
+        )
+
+    # endpoint_url is optional — only inject when non-empty (real AWS users don't set it)
+    endpoint_url: Optional[str] = config.get("endpoint_url") or os.getenv("AWS_S3_ENDPOINT") or None
+
+    region: str = config.get("region") or os.getenv("AWS_REGION") or _DEFAULT_REGION
+
+    return {
+        "access_key": access_key,
+        "secret_key": secret_key,
+        "endpoint_url": endpoint_url,
+        "region": region,
+    }
+
+
+def _build_boto3_kwargs(creds: Dict[str, Any]) -> Dict[str, Any]:
+    """Build the keyword arguments for boto3.resource / boto3.client."""
+    kwargs: Dict[str, Any] = {
+        "aws_access_key_id": creds["access_key"],
+        "aws_secret_access_key": creds["secret_key"],
+        "region_name": creds["region"],
+    }
+    if creds["endpoint_url"]:
+        kwargs["endpoint_url"] = creds["endpoint_url"]
+    return kwargs
+
+
+def create_s3_resource(config: Dict[str, Any]):
+    """Return a boto3 S3 resource (high-level API) for bucket/object access.
+
+    Works with AWS S3, MinIO, Cloudflare R2, and any S3-compatible service.
+    """
+    try:
+        import boto3
+    except ImportError as exc:
+        raise ImportError(
+            "boto3 is required for the S3 connector. "
+            "Install it with: pip install boto3"
+        ) from exc
+
+    creds = _resolve_credentials(config)
+    kwargs = _build_boto3_kwargs(creds)
+    logger.debug("Creating S3 resource with HMAC authentication (boto3)")
+    return boto3.resource("s3", **kwargs)
+
+
+def create_s3_client(config: Dict[str, Any]):
+    """Return a boto3 S3 low-level client.
+
+    Used for operations such as list_buckets() and get_object_acl().
+    """
+    try:
+        import boto3
+    except ImportError as exc:
+        raise ImportError(
+            "boto3 is required for the S3 connector. "
+            "Install it with: pip install boto3"
+        ) from exc
+
+    creds = _resolve_credentials(config)
+    kwargs = _build_boto3_kwargs(creds)
+    logger.debug("Creating S3 client with HMAC authentication (boto3)")
+    return boto3.client("s3", **kwargs)