waldiez 0.4.7__py3-none-any.whl → 0.4.9__py3-none-any.whl

waldiez/models/agents/{rag_user → rag_user_proxy}/retrieve_config.py

@@ -4,21 +4,21 @@
 
 import os
 from pathlib import Path
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Optional, Union
 
 from pydantic import Field, model_validator
 from typing_extensions import Annotated, Literal, Self
 
 from ...common import WaldiezBase, check_function, generate_function
-from .vector_db_config import WaldiezRagUserVectorDbConfig
+from .vector_db_config import WaldiezRagUserProxyVectorDbConfig
 
-WaldiezRagUserTask = Literal["code", "qa", "default"]
+WaldiezRagUserProxyTask = Literal["code", "qa", "default"]
 """Possible tasks for the retrieve chat."""
-WaldiezRagUserVectorDb = Literal["chroma", "pgvector", "mongodb", "qdrant"]
+WaldiezRagUserProxyVectorDb = Literal["chroma", "pgvector", "mongodb", "qdrant"]
 """Possible vector dbs for the retrieve chat."""
-WaldiezRagUserChunkMode = Literal["multi_lines", "one_line"]
+WaldiezRagUserProxyChunkMode = Literal["multi_lines", "one_line"]
 """Possible chunk modes for the retrieve chat."""
-WaldiezRagUserModels: Dict[WaldiezRagUserVectorDb, str] = {
+WaldiezRagUserProxyModels: dict[WaldiezRagUserProxyVectorDb, str] = {
     "chroma": "all-MiniLM-L6-v2",
     "mongodb": "all-MiniLM-L6-v2",
     "pgvector": "all-MiniLM-L6-v2",
@@ -26,8 +26,8 @@ WaldiezRagUserModels: Dict[WaldiezRagUserVectorDb, str] = {
 }
 
 CUSTOM_EMBEDDING_FUNCTION = "custom_embedding_function"
-CUSTOM_EMBEDDING_FUNCTION_ARGS: List[str] = []
-CUSTOM_EMBEDDING_FUNCTION_TYPES: Tuple[List[str], str] = (
+CUSTOM_EMBEDDING_FUNCTION_ARGS: list[str] = []
+CUSTOM_EMBEDDING_FUNCTION_TYPES: tuple[list[str], str] = (
     [],
     "Callable[..., Any]",
 )
@@ -48,7 +48,7 @@ CUSTOM_TEXT_SPLIT_FUNCTION_ARGS = [
 ]
 CUSTOM_TEXT_SPLIT_FUNCTION_TYPES = (
     ["str", "int", "str", "bool", "int"],
-    "List[str]",
+    "list[str]",
 )
 NOT_LOCAL = (
     "http://",
@@ -56,10 +56,14 @@ NOT_LOCAL = (
     "ftp://",
     "ftps://",
     "sftp://",
+    "hdfs",
+    "s3://",
+    "gs://",
+    "azure://",
 )
 
 
-class WaldiezRagUserRetrieveConfig(WaldiezBase):
+class WaldiezRagUserProxyRetrieveConfig(WaldiezBase):
     """RAG user agent.
 
     Attributes
@@ -74,7 +78,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
         The vector db for the retrieve chat.
     db_config : Annotated[WaldiezVectorDbConfig, Field]
         The config for the selected vector db.
-    docs_path : Optional[Union[str, List[str]]]
+    docs_path : Optional[Union[str, list[str]]]
         The path to the docs directory. It can also be the path to a single
         file, the url to a single file or a list of directories, files and
         urls. Default is None, which works only if the collection is already
@@ -147,7 +151,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
         A custom function to split a string into a list of strings. Default is
         None, will use the default function in autogen.retrieve_utils.
         split_text_to_chunks.
-    custom_text_types : Optional[List[str]]
+    custom_text_types : Optional[list[str]]
         A list of file types to be processed. Default is autogen.retrieve_utils.
         TEXT_FORMATS. This only applies to files under the directories in
         docs_path. Explicitly included files and urls will be chunked
@@ -180,9 +184,9 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
     """
 
     task: Annotated[
-        WaldiezRagUserTask,
+        WaldiezRagUserProxyTask,
         Field(
-            "default",
+            default="default",
             title="Task",
             description=(
                 "The task of the retrieve chat. "
@@ -193,25 +197,25 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "the response."
             ),
         ),
-    ]
+    ] = "default"
     vector_db: Annotated[
-        WaldiezRagUserVectorDb,
+        WaldiezRagUserProxyVectorDb,
         Field(
-            "chroma",
+            default="chroma",
             title="Vector DB",
             description="The vector db for the retrieve chat.",
         ),
-    ]
+    ] = "chroma"
     db_config: Annotated[
-        WaldiezRagUserVectorDbConfig,
+        WaldiezRagUserProxyVectorDbConfig,
         Field(
             title="DB Config",
             description="The config for the selected vector db.",
-            default_factory=WaldiezRagUserVectorDbConfig,
+            default_factory=WaldiezRagUserProxyVectorDbConfig,
         ),
     ]
     docs_path: Annotated[
-        Optional[Union[str, List[str]]],
+        Optional[Union[str, list[str]]],
         Field(
             default=None,
             title="Docs Path",
@@ -222,7 +226,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "only if the collection is already created."
             ),
         ),
-    ]
+    ] = None
     new_docs: Annotated[
         bool,
         Field(
@@ -236,7 +240,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "hash value of the content."
             ),
         ),
-    ]
+    ] = True
     model: Annotated[
         Optional[str],
         Field(
@@ -247,7 +251,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "we check for models linked to the agent."
             ),
         ),
-    ]
+    ] = None
     chunk_token_size: Annotated[
         Optional[int],
         Field(
@@ -259,7 +263,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "will be used."
             ),
         ),
-    ]
+    ] = None
     context_max_tokens: Annotated[
         Optional[int],
         Field(
@@ -271,9 +275,9 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "will be used."
             ),
         ),
-    ]
+    ] = None
     chunk_mode: Annotated[
-        WaldiezRagUserChunkMode,
+        WaldiezRagUserProxyChunkMode,
         Field(
             default="multi_lines",
             title="Chunk Mode",
@@ -283,8 +287,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "a default mode multi_lines will be used."
             ),
         ),
-    ]
-
+    ] = "multi_lines"
     must_break_at_empty_line: Annotated[
         bool,
         Field(
@@ -295,7 +298,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "If chunk_mode is 'one_line', this parameter will be ignored."
             ),
         ),
-    ]
+    ] = True
     use_custom_embedding: Annotated[
         bool,
         Field(
@@ -307,7 +310,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "provided."
             ),
         ),
-    ]
+    ] = False
     embedding_function: Annotated[
         Optional[str],
         Field(
@@ -322,7 +325,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "https://docs.trychroma.com/guides/embeddings."
             ),
         ),
-    ]
+    ] = None
     customized_prompt: Annotated[
         Optional[str],
         Field(
@@ -332,7 +335,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "The customized prompt for the retrieve chat. Default is None."
             ),
         ),
-    ]
+    ] = None
     customized_answer_prefix: Annotated[
         Optional[str],
         Field(
@@ -344,7 +347,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "not in the answer, Update Context will be triggered."
             ),
         ),
-    ]
+    ] = ""
     update_context: Annotated[
         bool,
         Field(
@@ -355,7 +358,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "retrieval. Default is True."
             ),
         ),
-    ]
+    ] = True
     collection_name: Annotated[
         str,
         Field(
@@ -366,7 +369,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "a default name autogen-docs will be used."
             ),
         ),
-    ]
+    ] = "autogen-docs"
     get_or_create: Annotated[
         bool,
         Field(
@@ -376,7 +379,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "Whether to get the collection if it exists. Default is False."
             ),
         ),
-    ]
+    ] = False
     overwrite: Annotated[
         bool,
         Field(
@@ -394,7 +397,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 " otherwise it raise a ValueError."
             ),
         ),
-    ]
+    ] = False
     use_custom_token_count: Annotated[
         bool,
         Field(
@@ -406,7 +409,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "custom_token_count_function should be provided."
             ),
         ),
-    ]
+    ] = False
     custom_token_count_function: Annotated[
         Optional[str],
         Field(
@@ -421,7 +424,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "tiktoken, which may not be accurate for non-OpenAI models."
             ),
         ),
-    ]
+    ] = None
     use_custom_text_split: Annotated[
         bool,
         Field(
@@ -433,7 +436,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "custom_text_split_function should be provided."
             ),
         ),
-    ]
+    ] = False
     custom_text_split_function: Annotated[
         Optional[str],
         Field(
@@ -445,9 +448,9 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "autogen.retrieve_utils.split_text_to_chunks."
             ),
         ),
-    ]
+    ] = None
     custom_text_types: Annotated[
-        Optional[List[str]],
+        Optional[list[str]],
         Field(
             default=None,
             title="Custom Text Types",
@@ -459,7 +462,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "chunked regardless of their types."
             ),
         ),
-    ]
+    ] = None
     recursive: Annotated[
         bool,
         Field(
@@ -470,7 +473,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "Default is True."
             ),
         ),
-    ]
+    ] = True
     distance_threshold: Annotated[
         float,
         Field(
@@ -482,7 +485,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "Will be ignored if < 0. Default is -1."
             ),
         ),
-    ]
+    ] = -1
     n_results: Annotated[
         Optional[int],
         Field(
@@ -494,7 +497,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
                 "Use None or <1 to return all results."
             ),
         ),
-    ]
+    ] = None
     _embedding_function_string: Optional[str] = None
 
     _token_count_function_string: Optional[str] = None
@@ -538,7 +541,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
         self,
         name_prefix: Optional[str] = None,
         name_suffix: Optional[str] = None,
-    ) -> Tuple[str, str]:
+    ) -> tuple[str, str]:
         """Generate the custom embedding function.
 
         Parameters
@@ -550,7 +553,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
 
         Returns
         -------
-        Tuple[str, str]
+        tuple[str, str]
             The custom embedding function and the function name.
         """
         function_name = CUSTOM_EMBEDDING_FUNCTION
@@ -572,7 +575,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
         self,
         name_prefix: Optional[str] = None,
         name_suffix: Optional[str] = None,
-    ) -> Tuple[str, str]:
+    ) -> tuple[str, str]:
         """Generate the custom token count function.
 
         Parameters
@@ -584,7 +587,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
 
         Returns
         -------
-        Tuple[str, str]
+        tuple[str, str]
             The custom token count function and the function name.
         """
         function_name = CUSTOM_TOKEN_COUNT_FUNCTION
@@ -606,7 +609,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
         self,
         name_prefix: Optional[str] = None,
         name_suffix: Optional[str] = None,
-    ) -> Tuple[str, str]:
+    ) -> tuple[str, str]:
         """Generate the custom text split function.
 
         Parameters
@@ -618,7 +621,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
 
         Returns
         -------
-        Tuple[str, str]
+        tuple[str, str]
             The custom text split function and the function name.
         """
         function_name = CUSTOM_TEXT_SPLIT_FUNCTION
@@ -715,35 +718,50 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
         """
         if not self.docs_path:
             return
-        # if urls or directories ok, if files they should resolve
+
+        # Normalize to list
         doc_paths = (
             [self.docs_path]
             if isinstance(self.docs_path, str)
             else self.docs_path
         )
-        paths: List[str] = []
+
+        validated_paths: list[str] = []
+
         for path in doc_paths:
-            resolved = path
-            is_remote, is_raw = is_remote_path(resolved)
-            if is_remote:
-                if not is_raw:
-                    resolved = f'r"{resolved}"'
-                if resolved not in paths:
-                    paths.append(resolved)
+            # Skip duplicates
+            if path in validated_paths:
                 continue
-            resolved = remove_file_scheme(resolved)
-            is_raw = resolved.startswith(("r'", 'r"'))
-            maybe_folder = string_represents_folder(resolved)
-            if maybe_folder:
-                if not is_raw:
-                    resolved = f'r"{resolved}"'
-                if resolved not in paths:
-                    paths.append(resolved)
+
+            # Check if it's a remote path
+            is_remote = is_remote_path(path)
+            if is_remote:
+                # Remote paths: ensure proper raw string wrapping if needed
+                content = extract_raw_string_content(path)
+                validated_paths.append(f'r"{content}"')
                 continue
-            resolved = resolve_path(resolved, is_raw, not maybe_folder)
-            if resolved not in paths:
-                paths.append(resolved)
-        self.docs_path = paths
+
+            # Handle local paths
+            # First remove any file:// scheme
+            cleaned_path = remove_file_scheme(path)
+            content = extract_raw_string_content(cleaned_path)
+
+            # Determine if it's likely a folder
+            is_folder = string_represents_folder(content)
+
+            if is_folder:
+                validated_paths.append(f'r"{content}"')
+            else:
+                # Files: resolve and validate existence
+                try:
+                    resolved_path = resolve_path(cleaned_path, must_exist=True)
+                    validated_paths.append(resolved_path)
+                except ValueError as e:
+                    raise ValueError(f"Invalid file path '{path}': {e}") from e
+
+        # remove dupes (but keep order)
+        validated_paths = list(dict.fromkeys(validated_paths))
+        self.docs_path = [path for path in validated_paths if path]
 
     @model_validator(mode="after")
     def validate_rag_user_data(self) -> Self:
@@ -756,7 +774,7 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
 
         Returns
         -------
-        WaldiezRagUserData
+        WaldiezRagUserProxyData
             The validated RAG user data.
         """
         self.validate_custom_embedding_function()
@@ -764,12 +782,35 @@ class WaldiezRagUserRetrieveConfig(WaldiezBase):
         self.validate_custom_text_split_function()
         self.validate_docs_path()
         if not self.db_config.model:
-            self.db_config.model = WaldiezRagUserModels[self.vector_db]
+            self.db_config.model = WaldiezRagUserProxyModels[self.vector_db]
         if isinstance(self.n_results, int) and self.n_results < 1:
             self.n_results = None
         return self
 
 
+def extract_raw_string_content(path: str) -> str:
+    """Extract content from potential raw string formats.
+
+    Parameters
+    ----------
+    path : str
+        The path that might be wrapped in raw string format.
+
+    Returns
+    -------
+    str
+        The actual content of the path, without raw string formatting.
+    """
+    # Handle r"..." and r'...'
+    if path.startswith(('r"', "r'")) and len(path) > 3:
+        quote = path[1]
+        if path.endswith(quote):
+            return path[2:-1]
+        # Handle malformed raw strings (missing end quote)
+        return path[2:]
+    return path
+
+
 def string_represents_folder(path: str) -> bool:
     """Check if a string represents a folder.
 
@@ -783,14 +824,27 @@ def string_represents_folder(path: str) -> bool:
     bool
         True if the path is likely a folder, False if it's likely a file.
     """
-    if path.endswith(os.path.sep):
-        return True
-    if os.path.isdir(path):
+    # Extract actual path content if wrapped
+    content = extract_raw_string_content(path)
+
+    # Explicit folder indicators
+    if content.endswith(("/", "\\", os.path.sep)):
         return True
-    return not os.path.splitext(path)[1]
 
+    # Check if it actually exists and is a directory
+    try:
+        if os.path.isdir(content):
+            return True
+    except (OSError, ValueError):  # pragma: no cover
+        pass
+
+    # Heuristic: no file extension likely means folder
+    # return not os.path.splitext(content)[1]
+    _, ext = os.path.splitext(path.rstrip("/\\"))
+    return not ext
 
-def is_remote_path(path: str) -> Tuple[bool, bool]:
+
+def is_remote_path(path: str) -> bool:
     """Check if a path is a remote path.
 
     Parameters
@@ -800,14 +854,14 @@ def is_remote_path(path: str) -> Tuple[bool, bool]:
 
     Returns
     -------
-    Tuple[bool, bool]
+    tuple[bool, bool]
         If the path is a remote path and if it's a raw string.
     """
-    is_raw = path.startswith(("r'", 'r"'))
+    content = extract_raw_string_content(path)
     for not_local in NOT_LOCAL:
-        if path.startswith((not_local, f'r"{not_local}', f"r'{not_local}")):
-            return True, is_raw
-    return False, is_raw
+        if content.startswith((not_local, f'r"{not_local}', f"r'{not_local}")):
+            return True
+    return False
 
 
 def remove_file_scheme(path: str) -> str:
@@ -823,54 +877,68 @@ def remove_file_scheme(path: str) -> str:
     str
         The path without the scheme.
     """
-    resolved = str(path)
-    while resolved.startswith('r"file://') and resolved.endswith('"'):
-        resolved = resolved[len('r"file://') : -1]
-    while resolved.startswith("r'file://") and resolved.endswith("'"):
-        resolved = resolved[len("r'file://") : -1]
-    while resolved.startswith("file://"):
-        resolved = resolved[len("file://") :]
-    return resolved
+    content = extract_raw_string_content(path)
+
+    # Remove file:// prefix
+    while content.startswith("file://"):
+        content = content[len("file://") :]
 
+    return f'r"{content}"'
 
-def resolve_path(path: str, is_raw: bool, must_exist: bool) -> str:
+
+def resolve_path(path: str, must_exist: bool) -> str:
     """Try to resolve a path.
 
     Parameters
     ----------
     path : str
         The path to resolve.
-    is_raw : bool
-        If the path is a raw string.
     must_exist : bool
         If the path must exist.
 
     Returns
     -------
-    Path
-        The resolved path.
+    str
+        The resolved path, potentially wrapped in raw string format.
 
     Raises
     ------
     ValueError
         If the path is not a valid local path.
     """
-    # pylint: disable=broad-except
-    path_string = path
-    if is_raw:
-        path_string = path[2:-1]
+    # Extract the actual path content
+    # if is_raw:
+    path_content = extract_raw_string_content(path)
+    # else:
+    #     path_content = path
+
+    # Handle JSON-escaped backslashes
+    if "\\\\" in path_content:  # pragma: no cover
+        path_content = path_content.replace("\\\\", "\\")
+    # pylint: disable=too-many-try-statements
     try:
-        resolved = Path(path_string).resolve()
-    except BaseException as error:  # pragma: no cover
-        # check if 'r'... is needed
-        raw_string = f'r"{path}"'
+        # Try to resolve the path
+        resolved = Path(path_content).resolve()
+
+        if must_exist and not resolved.exists():
+            raise ValueError(f"Path {path} does not exist.")
+
+        return f'r"{resolved}"'
+
+    except (
+        OSError,
+        UnicodeDecodeError,
+        ValueError,
+    ) as error:  # pragma: no cover
+        # Fallback: try as raw string for Windows compatibility
+        raw_version = f'r"{path_content}"'
         try:
-            Path(raw_string).resolve()
-        except BaseException:
+            # Test if the path can be resolved when treated as raw
+            resolved = Path(raw_version).resolve()
+            if must_exist and not resolved.exists():
+                raise ValueError(f"Path {path} does not exist.") from error
+            return raw_version
+        except Exception:
             raise ValueError(
-                f"Path {path} is not a valid local path."
+                f"Path {path} is not a valid local path: {error}"
             ) from error
-        return raw_string
-    if not resolved.exists() and must_exist:
-        raise ValueError(f"Path {path} does not exist.")
-    return f'r"{resolved}"'