alita-sdk 0.3.316__tar.gz → 0.3.317__tar.gz

{alita_sdk-0.3.316 → alita_sdk-0.3.317}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.316
+Version: 0.3.317
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedj27@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.316 → alita_sdk-0.3.317}/alita_sdk/runtime/langchain/document_loaders/AlitaExcelLoader.py

@@ -65,7 +65,7 @@ class AlitaExcelLoader(AlitaTableLoader):
         else:
             return df.to_string(index=False)
 
-    def read(self):
+    def read(self, lazy: bool = False):
         df = pd.read_excel(self.file_path, sheet_name=None)
         docs = []
         for key in df.keys():

{alita_sdk-0.3.316 → alita_sdk-0.3.317}/alita_sdk/runtime/langchain/document_loaders/constants.py

@@ -113,7 +113,7 @@ loaders_map = {
         'is_multimodal_processing': False,
         'kwargs': {
             'encoding': 'utf-8',
-            'raw_content': False,
+            'raw_content': True,
             'cleanse': False
         },
         'allowed_to_override': ['max_tokens']
@@ -122,7 +122,7 @@ loaders_map = {
         'class': AlitaExcelLoader,
         'is_multimodal_processing': False,
         'kwargs': {
-            'raw_content': False,
+            'raw_content': True,
             'cleanse': False
         },
         'allowed_to_override': ['max_tokens', LoaderProperties.LLM.value, LoaderProperties.PROMPT.value, LoaderProperties.PROMPT_DEFAULT.value]
@@ -131,7 +131,7 @@ loaders_map = {
         'class': AlitaExcelLoader,
         'is_multimodal_processing': False,
         'kwargs': {
-            'raw_content': False,
+            'raw_content': True,
             'cleanse': False
         },
         'allowed_to_override': ['max_tokens', LoaderProperties.LLM.value, LoaderProperties.PROMPT.value, LoaderProperties.PROMPT_DEFAULT.value]

{alita_sdk-0.3.316 → alita_sdk-0.3.317}/alita_sdk/runtime/tools/vectorstore.py

@@ -12,7 +12,6 @@ from alita_sdk.tools.vector_adapters.VectorStoreAdapter import VectorStoreAdapte
 from logging import getLogger
 
 from ..utils.logging import dispatch_custom_event
-from ..utils.utils import IndexerKeywords
 
 logger = getLogger(__name__)
 
@@ -248,10 +247,6 @@ class VectorStoreWrapper(BaseToolApiWrapper):
             tool_name="_clean_collection"
         )
 
-    def _get_indexed_data(self, collection_name: str):
-        """ Get all indexed data from vectorstore for non-code content """
-        return self.vector_adapter.get_indexed_data(self, collection_name)
-
     def _get_code_indexed_data(self, collection_suffix: str) -> Dict[str, Dict[str, Any]]:
         """ Get all indexed data from vectorstore for code content """
         return self.vector_adapter.get_code_indexed_data(self, collection_suffix)
@@ -308,26 +303,6 @@ class VectorStoreWrapper(BaseToolApiWrapper):
 
         return final_docs
 
-    def _reduce_non_code_duplicates(self, documents: Generator[Any, None, None], collection_suffix: str) -> List[Any]:
-        return self._reduce_duplicates(
-            documents,
-            collection_suffix,
-            self._get_indexed_data,
-            lambda doc: doc.metadata.get('id'),
-            lambda doc, idx: (
-                    doc.metadata.get('updated_on') and
-                    idx['metadata'].get('updated_on') and
-                    doc.metadata.get('updated_on') == idx['metadata'].get('updated_on')
-            ),
-            lambda idx_data, key: (
-                    idx_data[key]['all_chunks'] +
-                    [idx_data[dep_id]['id'] for dep_id in idx_data[key][IndexerKeywords.DEPENDENT_DOCS.value]] +
-                    [chunk_db_id for dep_id in idx_data[key][IndexerKeywords.DEPENDENT_DOCS.value]
-                     for chunk_db_id in idx_data[dep_id]['all_chunks']]
-            ),
-            log_msg="Verification of documents to index started"
-        )
-
     def _reduce_code_duplicates(self, documents: Generator[Any, None, None], collection_suffix: str) -> List[Any]:
         return self._reduce_duplicates(
             documents,
@@ -343,7 +318,7 @@ class VectorStoreWrapper(BaseToolApiWrapper):
             log_msg="Verification of code documents to index started"
         )
 
-    def index_documents(self, documents: Generator[Document, None, None], collection_suffix: str, progress_step: int = 20, clean_index: bool = True, is_code: bool = False):
+    def index_documents(self, documents: Generator[Document, None, None], collection_suffix: str, progress_step: int = 20, clean_index: bool = True, is_code: bool = True):
         """ Index documents in the vectorstore.
 
         Args:
@@ -374,8 +349,7 @@ class VectorStoreWrapper(BaseToolApiWrapper):
                 message="Filter for duplicates",
                 tool_name="index_documents")
             # remove duplicates based on metadata 'id' and 'updated_on' or 'commit_hash' fields
-            documents = self._reduce_code_duplicates(documents, collection_suffix) if is_code \
-                else self._reduce_non_code_duplicates(documents, collection_suffix)
+            documents = self._reduce_code_duplicates(documents, collection_suffix)
             self._log_tool_event(
                 message="All the duplicates were filtered out. Proceeding with indexing.",
                 tool_name="index_documents")

{alita_sdk-0.3.316 → alita_sdk-0.3.317}/alita_sdk/runtime/tools/vectorstore_base.py

@@ -256,6 +256,10 @@ class VectorStoreWrapperBase(BaseToolApiWrapper):
         progress_step = 20 if progress_step not in range(0, 100) else progress_step
         next_progress_point = progress_step
         for document in documents:
+            if not document.page_content:
+                # To avoid case when all documents have empty content
+                # See llm_processor.add_documents which exclude metadata of docs with empty content
+                continue
             documents_count += 1
             # logger.debug(f"Indexing document: {document}")
             try:

{alita_sdk-0.3.316 → alita_sdk-0.3.317}/alita_sdk/tools/browser/__init__.py

@@ -112,7 +112,8 @@ class BrowserToolkit(BaseToolkit):
             elif tool == 'google':
                 try:
                     google_api_wrapper = GoogleSearchAPIWrapper(
-                        **wrapper_payload_google
+                        google_api_key=wrapper_payload_google.get('google_api_key'),
+                        google_cse_id=wrapper_payload_google.get('google_cse_id')
                     )
                     tool_entry = GoogleSearchResults(api_wrapper=google_api_wrapper)
                     # rename the tool to avoid conflicts

{alita_sdk-0.3.316 → alita_sdk-0.3.317}/alita_sdk/tools/confluence/api_wrapper.py

@@ -1,24 +1,25 @@
-import re
-import logging
-import requests
-import json
 import base64
+import json
+import logging
+import re
 import traceback
-from typing import Optional, List, Any, Dict, Callable, Generator, Literal
 from json import JSONDecodeError
+from typing import Optional, List, Any, Dict, Callable, Generator, Literal
 
-from pydantic import Field, PrivateAttr, model_validator, create_model, SecretStr
-from tenacity import retry, stop_after_attempt, wait_exponential, before_sleep_log
-
+import requests
+from langchain_community.document_loaders.confluence import ContentFormat
 from langchain_core.documents import Document
-from langchain_core.tools import ToolException
 from langchain_core.messages import HumanMessage
+from langchain_core.tools import ToolException
 from markdownify import markdownify
-from langchain_community.document_loaders.confluence import ContentFormat
+from pydantic import Field, PrivateAttr, model_validator, create_model, SecretStr
+from tenacity import retry, stop_after_attempt, wait_exponential, before_sleep_log
 
-from ..elitea_base import BaseVectorStoreToolApiWrapper, extend_with_vector_tools
+from alita_sdk.tools.non_code_indexer_toolkit import NonCodeIndexerToolkit
+from alita_sdk.tools.utils.available_tools_decorator import extend_with_parent_available_tools
 from ..llm.img_utils import ImageDescriptionCache
 from ..utils import is_cookie_token, parse_cookie_string
+from ...runtime.utils.utils import IndexerKeywords
 
 logger = logging.getLogger(__name__)
 
@@ -171,7 +172,7 @@ def parse_payload_params(params: Optional[str]) -> Dict[str, Any]:
     return {}
 
 
-class ConfluenceAPIWrapper(BaseVectorStoreToolApiWrapper):
+class ConfluenceAPIWrapper(NonCodeIndexerToolkit):
     # Changed from PrivateAttr to Optional field with exclude=True
     client: Optional[Any] = Field(default=None, exclude=True)
     base_url: str
@@ -229,7 +230,7 @@ class ConfluenceAPIWrapper(BaseVectorStoreToolApiWrapper):
             client_instance._update_header(header, value)
 
         values['client'] = client_instance
-        return values
+        return super().validate_toolkit(values)
 
     def __unquote_confluence_space(self) -> str | None:
         if self.space:
@@ -843,13 +844,68 @@ class ConfluenceAPIWrapper(BaseVectorStoreToolApiWrapper):
             yield document
 
     def _process_document(self, document: Document) -> Generator[Document, None, None]:
-        attachments = self.get_page_attachments(document.metadata.get('id'))
-        if isinstance(attachments, str):
-            logger.info(f" {document.metadata.get('id')}: {attachments}")
-            return
-        for attachment in attachments:
-            # TODO need to retrive content from other fields/sources if both attachment.get('content', '') and attachment.get('llm_analysis', '') are None
-            yield Document(page_content=attachment.get('content', '') or attachment.get('llm_analysis', '') or '', metadata=attachment.get('metadata', {}))
+        try:
+            page_id = document.metadata.get('id')
+            attachments = self.client.get_attachments_from_content(page_id)
+            if not attachments or not attachments.get('results'):
+                return f"No attachments found for page ID {page_id}."
+
+            # Get attachment history for created/updated info
+            history_map = {}
+            for attachment in attachments['results']:
+                try:
+                    hist = self.client.history(attachment['id'])
+                    history_map[attachment['id']] = hist
+                except Exception as e:
+                    logger.warning(f"Failed to fetch history for attachment {attachment.get('title', '')}: {str(e)}")
+                    history_map[attachment['id']] = None
+
+            import re
+            for attachment in attachments['results']:
+                title = attachment.get('title', '')
+                file_ext = title.lower().split('.')[-1] if '.' in title else ''
+
+                media_type = attachment.get('metadata', {}).get('mediaType', '')
+                # Core metadata extraction with history
+                hist = history_map.get(attachment['id']) or {}
+                created_by = hist.get('createdBy', {}).get('displayName', '') if hist else attachment.get('creator', {}).get('displayName', '')
+                created_date = hist.get('createdDate', '') if hist else attachment.get('created', '')
+                last_updated = hist.get('lastUpdated', {}).get('when', '') if hist else ''
+
+                metadata = {
+                    'name': title,
+                    'size': attachment.get('extensions', {}).get('fileSize', None),
+                    'creator': created_by,
+                    'created': created_date,
+                    'updated': last_updated,
+                    'media_type': media_type,
+                    'labels': [label['name'] for label in
+                               attachment.get('metadata', {}).get('labels', {}).get('results', [])],
+                    'download_url': self.base_url.rstrip('/') + attachment['_links']['download'] if attachment.get(
+                        '_links', {}).get('download') else None
+                }
+
+                download_url = self.base_url.rstrip('/') + attachment['_links']['download']
+
+                try:
+                    resp = self.client.request(method="GET", path=download_url[len(self.base_url):], advanced_mode=True)
+                    if resp.status_code == 200:
+                        content = resp.content
+                    else:
+                        content = f"[Failed to download {download_url}: HTTP status code {resp.status_code}]"
+                except Exception as e:
+                    content = f"[Error downloading content: {str(e)}]"
+
+                if isinstance(content, str):
+                    yield Document(page_content=content, metadata=metadata)
+                else:
+                    yield Document(page_content="", metadata={
+                        **metadata,
+                        IndexerKeywords.CONTENT_FILE_NAME.value: f".{file_ext}",
+                        IndexerKeywords.CONTENT_IN_BYTES.value: content
+                    })
+        except Exception as e:
+            yield from ()
 
     def _download_image(self, image_url):
         """
@@ -1598,7 +1654,7 @@ class ConfluenceAPIWrapper(BaseVectorStoreToolApiWrapper):
             "bins_with_llm": (Optional[bool], Field(description="Use LLM for processing binary files.", default=False)),
         }
 
-    @extend_with_vector_tools
+    @extend_with_parent_available_tools
     def get_available_tools(self):
         # Confluence-specific tools
         confluence_tools = [

{alita_sdk-0.3.316 → alita_sdk-0.3.317}/alita_sdk/tools/jira/api_wrapper.py

@@ -13,7 +13,6 @@ from langchain_core.tools import ToolException
 from pydantic import Field, PrivateAttr, model_validator, create_model, SecretStr
 import requests
 
-from ..elitea_base import BaseVectorStoreToolApiWrapper, extend_with_vector_tools
 from ..llm.img_utils import ImageDescriptionCache
 from ..non_code_indexer_toolkit import NonCodeIndexerToolkit
 from ..utils import is_cookie_token, parse_cookie_string

{alita_sdk-0.3.316 → alita_sdk-0.3.317}/alita_sdk/tools/xray/api_wrapper.py

@@ -9,14 +9,9 @@ from langchain_core.tools import ToolException
 from pydantic import PrivateAttr, SecretStr, create_model, model_validator, Field
 from python_graphql_client import GraphqlClient
 
-from ..elitea_base import (
-    BaseVectorStoreToolApiWrapper,
-    extend_with_vector_tools,
-)
 from ..non_code_indexer_toolkit import NonCodeIndexerToolkit
 from ..utils.available_tools_decorator import extend_with_parent_available_tools
 from ...runtime.utils.utils import IndexerKeywords
-from ..utils.content_parser import file_extension_by_chunker
 
 try:
     from alita_sdk.runtime.langchain.interfaces.llm_processor import get_embeddings

{alita_sdk-0.3.316 → alita_sdk-0.3.317}/alita_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.316
+Version: 0.3.317
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedj27@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.316 → alita_sdk-0.3.317}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "alita_sdk"
-version = "0.3.316"
+version = "0.3.317"
 description = "SDK for building langchain agents using resources from Alita"
 readme = "README.md"
 requires-python = ">=3.10"