alita-sdk 0.3.379__py3-none-any.whl → 0.3.627__py3-none-any.whl

alita_sdk/tools/code/sonar/__init__.py

@@ -5,8 +5,9 @@ from pydantic import create_model, BaseModel, ConfigDict, Field
 from .api_wrapper import SonarApiWrapper
 from ...base.tool import BaseAction
 from ...elitea_base import filter_missconfigured_index_tools
-from ...utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
+from ...utils import clean_string, get_max_toolkit_length
 from ....configurations.sonar import SonarConfiguration
+from ....runtime.utils.constants import TOOLKIT_NAME_META, TOOL_NAME_META, TOOLKIT_TYPE_META
 
 name = "sonar"
 
@@ -21,15 +22,13 @@ def get_tools(tool):
 
 class SonarToolkit(BaseToolkit):
     tools: list[BaseTool] = []
-    toolkit_max_length: int = 0
 
     @staticmethod
     def toolkit_config_schema() -> BaseModel:
         selected_tools = {x['name']: x['args_schema'].schema() for x in SonarApiWrapper.model_construct().get_available_tools()}
-        SonarToolkit.toolkit_max_length = get_max_toolkit_length(selected_tools)
         return create_model(
             name,
-            sonar_project_name=(str, Field(description="Project name of the desired repository", json_schema_extra={'toolkit_name': True, 'max_toolkit_length': SonarToolkit.toolkit_max_length})),
+            sonar_project_name=(str, Field(description="Project name of the desired repository")),
             sonar_configuration=(SonarConfiguration, Field(description="Sonar Configuration", json_schema_extra={'configuration_types': ['sonar']})),
             selected_tools=(List[Literal[tuple(selected_tools)]], Field(default=[], json_schema_extra={'args_schemas': selected_tools})),
             __config__=ConfigDict(json_schema_extra=
@@ -55,15 +54,19 @@ class SonarToolkit(BaseToolkit):
         sonar_api_wrapper = SonarApiWrapper(**wrapper_payload)
         available_tools = sonar_api_wrapper.get_available_tools()
         tools = []
-        prefix = clean_string(toolkit_name, SonarToolkit.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         for tool in available_tools:
             if selected_tools and tool["name"] not in selected_tools:
                 continue
+            description = tool["description"]
+            if toolkit_name:
+                description = f"Toolkit: {toolkit_name}\n{description}"
+            description = description[:1000]
             tools.append(BaseAction(
                 api_wrapper=sonar_api_wrapper,
-                name=prefix + tool["name"],
-                description=tool["description"],
-                args_schema=tool["args_schema"]
+                name=tool["name"],
+                description=description,
+                args_schema=tool["args_schema"],
+                metadata={TOOLKIT_NAME_META: toolkit_name, TOOLKIT_TYPE_META: name, TOOL_NAME_META: tool["name"]} if toolkit_name else {TOOL_NAME_META: tool["name"]}
             ))
         return cls(tools=tools)
 

alita_sdk/tools/code_indexer_toolkit.py

@@ -1,5 +1,6 @@
 import ast
 import fnmatch
+import json
 import logging
 from typing import Optional, List, Generator
 
@@ -8,20 +9,20 @@ from langchain_core.tools import ToolException
 from pydantic import Field
 
 from alita_sdk.tools.base_indexer_toolkit import BaseIndexerToolkit
-from .chunkers.code.codeparser import parse_code_files_for_db
 
 logger = logging.getLogger(__name__)
 
 
 class CodeIndexerToolkit(BaseIndexerToolkit):
     def _get_indexed_data(self, index_name: str):
+        self._ensure_vectorstore_initialized()
         if not self.vector_adapter:
             raise ToolException("Vector adapter is not initialized. "
                              "Check your configuration: embedding_model and vectorstore_type.")
         return self.vector_adapter.get_code_indexed_data(self, index_name)
 
     def key_fn(self, document: Document):
-        return document.metadata.get('id')
+        return document.metadata.get("filename")
 
     def compare_fn(self, document: Document, idx_data):
         return (document.metadata.get('commit_hash') and
@@ -37,16 +38,18 @@ class CodeIndexerToolkit(BaseIndexerToolkit):
             branch: Optional[str] = None,
             whitelist: Optional[List[str]] = None,
             blacklist: Optional[List[str]] = None,
+            chunking_config: Optional[dict] = None,
             **kwargs) -> Generator[Document, None, None]:
         """Index repository files in the vector store using code parsing."""
         yield from self.loader(
             branch=branch,
             whitelist=whitelist,
-            blacklist=blacklist
+            blacklist=blacklist,
+            chunking_config=chunking_config
         )
 
     def _extend_data(self, documents: Generator[Document, None, None]):
-        yield from parse_code_files_for_db(documents)
+        yield from documents
 
     def _index_tool_params(self):
         """Return the parameters for indexing data."""
@@ -65,26 +68,55 @@ class CodeIndexerToolkit(BaseIndexerToolkit):
     def loader(self,
                branch: Optional[str] = None,
                whitelist: Optional[List[str]] = None,
-               blacklist: Optional[List[str]] = None) -> Generator[Document, None, None]:
+               blacklist: Optional[List[str]] = None,
+               chunked: bool = True,
+               chunking_config: Optional[dict] = None) -> Generator[Document, None, None]:
         """
-        Generates file content from a branch, respecting whitelist and blacklist patterns.
+        Generates Documents from files in a branch, respecting whitelist and blacklist patterns.
     
         Parameters:
         - branch (Optional[str]): Branch for listing files. Defaults to the current branch if None.
         - whitelist (Optional[List[str]]): File extensions or paths to include. Defaults to all files if None.
         - blacklist (Optional[List[str]]): File extensions or paths to exclude. Defaults to no exclusions if None.
+        - chunked (bool): If True (default), applies universal chunker based on file type.
+                         If False, returns raw Documents without chunking.
+        - chunking_config (Optional[dict]): Chunking configuration by file extension
     
         Returns:
-        - generator: Yields content from files matching the whitelist but not the blacklist.
+        - generator: Yields Documents from files matching the whitelist but not the blacklist.
+                    Each document has exactly the key 'filename' in metadata, which is used as an ID
+                    for further operations (indexing, deduplication, and retrieval).
     
         Example:
         # Use 'feature-branch', include '.py' files, exclude 'test_' files
-        file_generator = loader(branch='feature-branch', whitelist=['*.py'], blacklist=['*test_*'])
+        for doc in loader(branch='feature-branch', whitelist=['*.py'], blacklist=['*test_*']):
+            print(doc.page_content)
     
         Notes:
         - Whitelist and blacklist use Unix shell-style wildcards.
         - Files must match the whitelist and not the blacklist to be included.
+        - Each document MUST have exactly the key 'filename' in metadata. This key is used as an ID
+          for further operations such as indexing, deduplication, and retrieval.
+        - When chunked=True:
+          - .md files → markdown chunker (header-based splitting)
+          - .py/.js/.ts/etc → code parser (TreeSitter-based)
+          - .json files → JSON chunker
+          - other files → default text chunker
         """
+        import hashlib
+    
+        # Auto-include extensions from chunking_config if whitelist is specified
+        # This allows chunking config to work without manually adding extensions to whitelist
+        if chunking_config and whitelist:
+            for ext_pattern in chunking_config.keys():
+                # Normalize extension pattern (both ".cbl" and "*.cbl" should work)
+                normalized = ext_pattern if ext_pattern.startswith('*') else f'*{ext_pattern}'
+                if normalized not in whitelist:
+                    whitelist.append(normalized)
+                    self._log_tool_event(
+                        message=f"Auto-included extension '{normalized}' from chunking_config",
+                        tool_name="loader"
+                    )
     
         _files = self.__handle_get_files("", self.__get_branch(branch))
         self._log_tool_event(message="Listing files in branch", tool_name="loader")
@@ -102,32 +134,60 @@ class CodeIndexerToolkit(BaseIndexerToolkit):
                         or any(file_path.endswith(f'.{pattern}') for pattern in blacklist))
             return False
     
-        def file_content_generator():
+        def raw_document_generator() -> Generator[Document, None, None]:
+            """Yields raw Documents without chunking."""
             self._log_tool_event(message="Reading the files", tool_name="loader")
-            # log the progress of file reading
             total_files = len(_files)
+            processed = 0
+            
             for idx, file in enumerate(_files, 1):
                 if is_whitelisted(file) and not is_blacklisted(file):
-                    # read file ONLY if it matches whitelist and does not match blacklist
                     try:
                         file_content = self._read_file(file, self.__get_branch(branch))
                     except Exception as e:
                         logger.error(f"Failed to read file {file}: {e}")
-                        file_content = ""
+                        continue
+                    
                     if not file_content:
-                        # empty file, skip
                         continue
-                    # hash the file content to ensure uniqueness
-                    import hashlib
+                    
+                    # Ensure file content is a string
+                    if isinstance(file_content, bytes):
+                        file_content = file_content.decode("utf-8", errors="ignore")
+                    elif isinstance(file_content, dict) and file.endswith('.json'):
+                        file_content = json.dumps(file_content)
+                    elif not isinstance(file_content, str):
+                        file_content = str(file_content)
+                    
+                    # Hash the file content for uniqueness tracking
                     file_hash = hashlib.sha256(file_content.encode("utf-8")).hexdigest()
-                    yield {"file_name": file,
-                           "file_content": file_content,
-                           "commit_hash": file_hash}
+                    processed += 1
+                    
+                    yield Document(
+                        page_content=file_content,
+                        metadata={
+                            'file_path': file,
+                            'filename': file,
+                            'source': file,
+                            'commit_hash': file_hash,
+                        }
+                    )
+                
                 if idx % 10 == 0 or idx == total_files:
-                    self._log_tool_event(message=f"{idx} out of {total_files} files have been read", tool_name="loader")
-            self._log_tool_event(message=f"{len(_files)} have been read", tool_name="loader")
-    
-        return file_content_generator()
+                    self._log_tool_event(
+                        message=f"{idx} out of {total_files} files checked, {processed} matched",
+                        tool_name="loader"
+                    )
+            
+            self._log_tool_event(message=f"{processed} files loaded", tool_name="loader")
+
+        if not chunked:
+            # Return raw documents without chunking
+            return raw_document_generator()
+        
+        # Apply universal chunker based on file type
+        from .chunkers.universal_chunker import universal_chunker
+        return universal_chunker(raw_document_generator())
 
     def __handle_get_files(self, path: str, branch: str):
         """

alita_sdk/tools/confluence/__init__.py

@@ -6,14 +6,15 @@ from ..base.tool import BaseAction
 from pydantic import create_model, BaseModel, ConfigDict, Field
 
 from ..elitea_base import filter_missconfigured_index_tools
-from ..utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length, parse_list, check_connection_response
+from ..utils import clean_string, get_max_toolkit_length, parse_list, check_connection_response
 from ...configurations.confluence import ConfluenceConfiguration
 from ...configurations.pgvector import PgVectorConfiguration
 import requests
+from ...runtime.utils.constants import TOOLKIT_NAME_META, TOOL_NAME_META, TOOLKIT_TYPE_META
 
 name = "confluence"
 
-def get_tools(tool):
+def get_toolkit(tool):
     return ConfluenceToolkit().get_toolkit(
         selected_tools=tool['settings'].get('selected_tools', []),
         space=tool['settings'].get('space', None),
@@ -33,18 +34,19 @@ def get_tools(tool):
         doctype='doc',
         embedding_model=tool['settings'].get('embedding_model'),
         vectorstore_type="PGVector"
-    ).get_tools()
+    )
+
+def get_tools(tool):
+    return get_toolkit(tool).get_tools()
 
 
 class ConfluenceToolkit(BaseToolkit):
     tools: List[BaseTool] = []
-    toolkit_max_length: int = 0
 
     @staticmethod
     def toolkit_config_schema() -> BaseModel:
         selected_tools = {x['name']: x['args_schema'].schema() for x in
                           ConfluenceAPIWrapper.model_construct().get_available_tools()}
-        ConfluenceToolkit.toolkit_max_length = get_max_toolkit_length(selected_tools)
 
         @check_connection_response
         def check_connection(self):
@@ -67,19 +69,18 @@ class ConfluenceToolkit(BaseToolkit):
 
         model = create_model(
             name,
-            space=(str, Field(description="Space", json_schema_extra={'toolkit_name': True,
-                                                                                    'max_toolkit_length': ConfluenceToolkit.toolkit_max_length})),
+            space=(str, Field(description="Space")),
             cloud=(bool, Field(description="Hosting Option", json_schema_extra={'configuration': True})),
-            limit=(int, Field(description="Pages limit per request", default=5)),
+            limit=(int, Field(description="Pages limit per request", default=5, gt=0)),
             labels=(Optional[str], Field(
                 description="List of comma separated labels used for labeling of agent's created or updated entities",
                 default=None,
                 examples="alita,elitea;another-label"
             )),
-            max_pages=(int, Field(description="Max total pages", default=10)),
-            number_of_retries=(int, Field(description="Number of retries", default=2)),
-            min_retry_seconds=(int, Field(description="Min retry, sec", default=10)),
-            max_retry_seconds=(int, Field(description="Max retry, sec", default=60)),
+            max_pages=(int, Field(description="Max total pages", default=10, gt=0)),
+            number_of_retries=(int, Field(description="Number of retries", default=2, ge=0)),
+            min_retry_seconds=(int, Field(description="Min retry, sec", default=10, ge=0)),
+            max_retry_seconds=(int, Field(description="Max retry, sec", default=60, ge=0)),
             # optional field for custom headers as dictionary
             custom_headers=(Optional[dict], Field(description="Custom headers for API requests", default={})),
             confluence_configuration=(ConfluenceConfiguration, Field(description="Confluence Configuration", json_schema_extra={'configuration_types': ['confluence']})),
@@ -115,18 +116,23 @@ class ConfluenceToolkit(BaseToolkit):
             **(kwargs.get('pgvector_configuration') or {}),
         }
         confluence_api_wrapper = ConfluenceAPIWrapper(**wrapper_payload)
-        prefix = clean_string(toolkit_name, ConfluenceToolkit.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         available_tools = confluence_api_wrapper.get_available_tools()
         tools = []
         for tool in available_tools:
             if selected_tools:
                 if tool["name"] not in selected_tools:
                     continue
+            description = tool["description"]
+            if toolkit_name:
+                description = f"Toolkit: {toolkit_name}\n{description}"
+            description = f"Confluence space: {confluence_api_wrapper.space}\n{description}"
+            description = description[:1000]
             tools.append(BaseAction(
                 api_wrapper=confluence_api_wrapper,
-                name=prefix + tool["name"],
-                description=f"Confluence space: {confluence_api_wrapper.space}" + tool["description"],
-                args_schema=tool["args_schema"]
+                name=tool["name"],
+                description=description,
+                args_schema=tool["args_schema"],
+                metadata={TOOLKIT_NAME_META: toolkit_name, TOOLKIT_TYPE_META: name, TOOL_NAME_META: tool["name"]} if toolkit_name else {TOOL_NAME_META: tool["name"]}
             ))
         return cls(tools=tools)
 

alita_sdk/tools/confluence/api_wrapper.py

@@ -7,12 +7,14 @@ from json import JSONDecodeError
 from typing import Optional, List, Any, Dict, Callable, Generator, Literal
 
 import requests
+from atlassian.errors import ApiError
 from langchain_community.document_loaders.confluence import ContentFormat
 from langchain_core.documents import Document
 from langchain_core.messages import HumanMessage
 from langchain_core.tools import ToolException
 from markdownify import markdownify
 from pydantic import Field, PrivateAttr, model_validator, create_model, SecretStr
+from requests import HTTPError
 from tenacity import retry, stop_after_attempt, wait_exponential, before_sleep_log
 
 from alita_sdk.tools.non_code_indexer_toolkit import NonCodeIndexerToolkit
@@ -194,6 +196,7 @@ class ConfluenceAPIWrapper(NonCodeIndexerToolkit):
     keep_markdown_format: Optional[bool] = True
     ocr_languages: Optional[str] = None
     keep_newlines: Optional[bool] = True
+    _errors: Optional[list[str]] = None
     _image_cache: ImageDescriptionCache = PrivateAttr(default_factory=ImageDescriptionCache)
 
     @model_validator(mode='before')
@@ -477,28 +480,78 @@ class ConfluenceAPIWrapper(NonCodeIndexerToolkit):
         """Gets pages with specific label in the Confluence space."""
 
         start = 0
-        pages_info = []
-        for _ in range((self.max_pages + self.limit - 1) // self.limit):
-            pages = self.client.get_all_pages_by_label(label, start=start,
-                                                       limit=self.limit)  # , expand="body.view.value"
+        pages_info: List[Dict[str, Any]] = []
+        seen_ids: set[str] = set()
+
+        # Use a while-loop driven by unique pages collected and
+        # presence of additional results instead of a fixed number
+        # of iterations based purely on max_pages/limit.
+        while len(pages_info) < (self.max_pages or 0):
+            pages = self.client.get_all_pages_by_label(
+                label,
+                start=start,
+                limit=self.limit,
+            )  # , expand="body.view.value"
             if not pages:
                 break
 
-            pages_info += [{
-                'page_id': page.metadata['id'],
-                'page_title': page.metadata['title'],
-                'page_url': page.metadata['source'],
-                'content': page.page_content
-            } for page in self.get_pages_by_id([page["id"] for page in pages])]
+            # Collect only ids we haven't processed yet to avoid
+            # calling get_page_by_id multiple times for the same
+            # Confluence page.
+            new_ids: List[str] = []
+            for p in pages:
+                page_id = p["id"] if isinstance(p, dict) else getattr(p, "id", None)
+                if page_id is None:
+                    continue
+                if page_id in seen_ids:
+                    continue
+                seen_ids.add(page_id)
+                new_ids.append(page_id)
+
+            if new_ids:
+                for page in self.get_pages_by_id(new_ids):
+                    meta = getattr(page, "metadata", {}) or {}
+                    page_id = meta.get("id")
+                    page_title = meta.get("title")
+                    page_url = meta.get("source")
+                    content = getattr(page, "page_content", None)
+
+                    if page_id is None:
+                        continue
+
+                    pages_info.append(
+                        {
+                            "page_id": page_id,
+                            "page_title": page_title,
+                            "page_url": page_url,
+                            "content": content,
+                        }
+                    )
+
+                    # Respect max_pages on unique pages collected.
+                    if len(pages_info) >= (self.max_pages or 0):
+                        break
+
+            # Advance the offset by the requested page size.
             start += self.limit
-        return pages_info
+
+            # Defensive break: if the API returns fewer items than
+            # requested, there are likely no more pages to fetch.
+            if len(pages) < self.limit:
+                break
+
+        # Slice as an extra safety net in case of any race conditions
+        # around the max_pages guard in the loop above.
+        return pages_info[: (self.max_pages or len(pages_info))]
 
     def is_public_page(self, page: dict) -> bool:
         """Check if a page is publicly accessible."""
         restrictions = self.client.get_all_restrictions_for_content(page["id"])
 
         return (
-                page["status"] == "current"
+                (page["status"] == "current"
+                # allow user to see archived content if needed
+                 or page["status"] == "archived")
                 and not restrictions["read"]["restrictions"]["user"]["results"]
                 and not restrictions["read"]["restrictions"]["group"]["results"]
         )
@@ -518,18 +571,35 @@ class ConfluenceAPIWrapper(NonCodeIndexerToolkit):
                 ),
                 before_sleep=before_sleep_log(logger, logging.WARNING),
             )(self.client.get_page_by_id)
-            page = get_page(
-                page_id=page_id, expand=f"{self.content_format.value},version"
-            )
-            if not self.include_restricted_content and not self.is_public_page(page):
-                continue
+            try:
+                page = get_page(
+                    page_id=page_id, expand=f"{self.content_format.value},version"
+                )
+            except (ApiError, HTTPError) as e:
+                logger.error(f"Error fetching page with ID {page_id}: {e}")
+                page_content_temp = f"Confluence API Error: cannot fetch the page with ID {page_id}: {e}"
+                # store errors
+                if self._errors is None:
+                    self._errors = []
+                self._errors.append(page_content_temp)
+                return Document(page_content=page_content_temp,
+                                metadata={})
+            # TODO: update on toolkit advanced settings level as a separate feature
+            # if not self.include_restricted_content and not self.is_public_page(page):
+            #     continue
             yield self.process_page(page, skip_images)
 
+    def _log_errors(self):
+        """ Log errors encountered during toolkit execution. """
+        if self._errors:
+            logger.info(f"Errors encountered during toolkit execution: {self._errors}")
+
     def read_page_by_id(self, page_id: str, skip_images: bool = False):
         """Reads a page by its id in the Confluence space. If id is not available, but there is a title - use get_page_id first."""
         result = list(self.get_pages_by_id([page_id], skip_images))
         if not result:
-            "Page not found"
+            return f"Pages not found. Errors: {self._errors}" if self._errors \
+                else "Pages not found or you do not have access to them."
         return result[0].page_content
         # return self._strip_base64_images(result[0].page_content) if skip_images else result[0].page_content
 
@@ -550,11 +620,18 @@ class ConfluenceAPIWrapper(NonCodeIndexerToolkit):
     def _process_search(self, cql, skip_images: bool = False):
         start = 0
         pages_info = []
+        seen_ids: set = set()  # Track seen page IDs to avoid duplicates
         for _ in range((self.max_pages + self.limit - 1) // self.limit):
             pages = self.client.cql(cql, start=start, limit=self.limit).get("results", [])
             if not pages:
                 break
-            page_ids = [page['content']['id'] for page in pages]
+            # Deduplicate page IDs before processing
+            page_ids = []
+            for page in pages:
+                page_id = page['content']['id']
+                if page_id not in seen_ids:
+                    seen_ids.add(page_id)
+                    page_ids.append(page_id)
             for page in self.get_pages_by_id(page_ids, skip_images):
                 page_info = {
                     'content': page.page_content,
@@ -874,14 +951,14 @@ class ConfluenceAPIWrapper(NonCodeIndexerToolkit):
 
                     # Re-verify extension filters
                     # Check if file should be skipped based on skip_extensions
-                    if any(re.match(pattern.replace('*', '.*') + '$', title, re.IGNORECASE)
+                    if any(re.match(re.escape(pattern).replace(r'\*', '.*') + '$', title, re.IGNORECASE)
                            for pattern in self._skip_extensions):
                         continue
 
                     # Check if file should be included based on include_extensions
                     # If include_extensions is empty, process all files (that weren't skipped)
                     if self._include_extensions and not (
-                    any(re.match(pattern.replace('*', '.*') + '$', title, re.IGNORECASE)
+                    any(re.match(re.escape(pattern).replace(r'\*', '.*') + '$', title, re.IGNORECASE)
                         for pattern in self._include_extensions)):
                         continue
 
@@ -892,6 +969,9 @@ class ConfluenceAPIWrapper(NonCodeIndexerToolkit):
                     created_date = hist.get('createdDate', '') if hist else attachment.get('created', '')
                     last_updated = hist.get('lastUpdated', {}).get('when', '') if hist else ''
 
+                    attachment_path = attachment['_links']['download'] if attachment.get(
+                            '_links', {}).get('download') else ''
+                    download_url = self.client.url.rstrip('/') + attachment_path
                     metadata = {
                         'name': title,
                         'size': attachment.get('extensions', {}).get('fileSize', None),
@@ -901,14 +981,10 @@ class ConfluenceAPIWrapper(NonCodeIndexerToolkit):
                         'media_type': media_type,
                         'labels': [label['name'] for label in
                                    attachment.get('metadata', {}).get('labels', {}).get('results', [])],
-                        'download_url': self.base_url.rstrip('/') + attachment['_links']['download'] if attachment.get(
-                            '_links', {}).get('download') else None
+                        'download_url': download_url
                     }
-
-                    download_url = self.base_url.rstrip('/') + attachment['_links']['download']
-
                     try:
-                        resp = self.client.request(method="GET", path=download_url[len(self.base_url):], advanced_mode=True)
+                        resp = self.client.request(method="GET", path=attachment_path, advanced_mode=True)
                         if resp.status_code == 200:
                             content = resp.content
                         else:
@@ -1661,8 +1737,8 @@ class ConfluenceAPIWrapper(NonCodeIndexerToolkit):
             "page_ids": (Optional[List[str]], Field(description="List of page IDs to retrieve.", default=None)),
             "label": (Optional[str], Field(description="Label to filter pages.", default=None)),
             "cql": (Optional[str], Field(description="CQL query to filter pages.", default=None)),
-            "limit": (Optional[int], Field(description="Limit the number of results.", default=10)),
-            "max_pages": (Optional[int], Field(description="Maximum number of pages to retrieve.", default=1000)),
+            "limit": (Optional[int], Field(description="Limit the number of results.", default=10, gt=0)),
+            "max_pages": (Optional[int], Field(description="Maximum number of pages to retrieve.", default=1000, gt=0)),
             "include_restricted_content": (Optional[bool], Field(description="Include restricted content.", default=False)),
             "include_archived_content": (Optional[bool], Field(description="Include archived content.", default=False)),
             "include_attachments": (Optional[bool], Field(description="Include attachments.", default=False)),
@@ -1798,4 +1874,5 @@ class ConfluenceAPIWrapper(NonCodeIndexerToolkit):
                 "description": self.get_page_attachments.__doc__,
                 "args_schema": GetPageAttachmentsInput,
             }
-        ]
+        ]
+

alita_sdk/tools/confluence/loader.py

@@ -3,6 +3,7 @@ from typing import Optional, List
 from logging import getLogger
 
 import requests
+from langchain_core.documents import Document
 
 logger = getLogger(__name__)
 from PIL import Image
@@ -47,7 +48,8 @@ class AlitaConfluenceLoader(ConfluenceLoader):
                 del kwargs[key]
             except:
                 pass
-        self.base_url = kwargs.get('url')
+        # utilize adjusted URL from Confluence instance for base_url
+        self.base_url = confluence_client.url
         self.space_key = kwargs.get('space_key')
         self.page_ids = kwargs.get('page_ids')
         self.label = kwargs.get('label')
@@ -107,7 +109,8 @@ class AlitaConfluenceLoader(ConfluenceLoader):
         texts = []
         for attachment in attachments:
             media_type = attachment["metadata"]["mediaType"]
-            absolute_url = self.base_url + attachment["_links"]["download"]
+            # utilize adjusted URL from Confluence instance for attachment download URL
+            absolute_url = self.confluence.url + attachment["_links"]["download"]
             title = attachment["title"]
             try:
                 if media_type == "application/pdf":
@@ -193,6 +196,15 @@ class AlitaConfluenceLoader(ConfluenceLoader):
         else:
             return super().process_image(link, ocr_languages)
 
+    def process_page(self, page: dict, include_attachments: bool, include_comments: bool, include_labels: bool,
+                     content_format: ContentFormat, ocr_languages: Optional[str] = None,
+                     keep_markdown_format: Optional[bool] = False, keep_newlines: bool = False) -> Document:
+        if not page.get("title"):
+            # if 'include_restricted_content' set to True, draft pages are loaded and can have no title
+            page["title"] = "Untitled"
+        return super().process_page(page, include_attachments, include_comments, include_labels, content_format,
+                                    ocr_languages, keep_markdown_format, keep_newlines)
+
     # TODO review usage
     # def process_svg(
     #         self,

alita_sdk/tools/custom_open_api/__init__.py

@@ -5,7 +5,8 @@ from pydantic import create_model, BaseModel, ConfigDict, Field
 
 from .api_wrapper import OpenApiWrapper
 from ..base.tool import BaseAction
-from ..utils import clean_string, TOOLKIT_SPLITTER
+from ..utils import clean_string
+from ...runtime.utils.constants import TOOLKIT_NAME_META, TOOL_NAME_META, TOOLKIT_TYPE_META
 
 name = "openapi"
 
@@ -43,15 +44,21 @@ class OpenApiToolkit(BaseToolkit):
         openapi_api_wrapper = OpenApiWrapper(**kwargs)
         available_tools = openapi_api_wrapper.get_available_tools()
         tools = []
-        prefix = clean_string(toolkit_name + TOOLKIT_SPLITTER) if toolkit_name else ''
+        # Use clean toolkit name for context (max 1000 chars in description)
+        toolkit_context = f" [Toolkit: {clean_string(toolkit_name)}]" if toolkit_name else ''
         for tool in available_tools:
             if selected_tools and tool["name"] not in selected_tools:
                 continue
+            # Add toolkit context to description with character limit
+            description = tool["description"]
+            if toolkit_context and len(description + toolkit_context) <= 1000:
+                description = description + toolkit_context
             tools.append(BaseAction(
                 api_wrapper=openapi_api_wrapper,
-                name=prefix + tool["name"],
-                description=tool["description"],
-                args_schema=tool["args_schema"]
+                name=tool["name"],
+                description=description,
+                args_schema=tool["args_schema"],
+                metadata={TOOLKIT_NAME_META: toolkit_name, TOOLKIT_TYPE_META: name, TOOL_NAME_META: tool["name"]} if toolkit_name else {TOOL_NAME_META: tool["name"]}
             ))
         return cls(tools=tools)