alita-sdk 0.3.253__py3-none-any.whl → 0.3.254__py3-none-any.whl

alita_sdk/runtime/clients/client.py

@@ -602,16 +602,22 @@ class AlitaClient:
         import logging
         logger = logging.getLogger(__name__)
         toolkit_config_parsed_json = None
+        events_dispatched = []
+
         try:
             toolkit_config_type = toolkit_config.get('type')
-            toolkit_class = get_available_toolkit_models().get(toolkit_config_type)['toolkit_class']
-            toolkit_config_model_class = toolkit_class.toolkit_config_schema()
-            toolkit_config_validated_settings = toolkit_config_model_class(
-                **toolkit_config.get('settings', {})
-            ).model_dump(mode='json')
-
+            available_toolkit_models = get_available_toolkit_models().get(toolkit_config_type)
             toolkit_config_parsed_json = deepcopy(toolkit_config)
-            toolkit_config_parsed_json['settings'] = toolkit_config_validated_settings
+            if available_toolkit_models:
+                toolkit_class = available_toolkit_models['toolkit_class']
+                toolkit_config_model_class = toolkit_class.toolkit_config_schema()
+                toolkit_config_validated_settings = toolkit_config_model_class(
+                    **toolkit_config.get('settings', {})
+                ).model_dump(mode='json')
+                toolkit_config_parsed_json['settings'] = toolkit_config_validated_settings
+            else:
+                logger.warning(f"Toolkit type '{toolkit_config_type}' is skipping model validation")
+                toolkit_config_parsed_json['settings'] = None
         except Exception as toolkit_config_error:
             logger.error(f"Failed to validate toolkit configuration: {str(toolkit_config_error)}")
             return {
@@ -635,7 +641,6 @@ class AlitaClient:
             # Create RunnableConfig for callback support
             config = None
             callbacks = []
-            events_dispatched = []
 
             if runtime_config:
                 callbacks = runtime_config.get('callbacks', [])

alita_sdk/runtime/langchain/document_loaders/AlitaJSONLoader.py

@@ -0,0 +1,73 @@
+import json
+from typing import Iterator
+
+from langchain_core.documents import Document
+
+from langchain_community.document_loaders.base import BaseLoader
+from langchain_community.document_loaders.helpers import detect_file_encodings
+from langchain_core.tools import ToolException
+from langchain_text_splitters import RecursiveJsonSplitter
+
+
+class AlitaJSONLoader(BaseLoader):
+
+    def __init__(self, **kwargs):
+        """Initialize with file path."""
+        if kwargs.get('file_path'):
+            self.file_path = kwargs['file_path']
+        elif kwargs.get('file_content'):
+            self.file_content = kwargs['file_content']
+            self.file_name = kwargs['file_name']
+        else:
+            raise ToolException("'file_path' or 'file_content' parameter should be provided.")
+        self.encoding = kwargs.get('encoding', 'utf-8')
+        self.autodetect_encoding = kwargs.get('autodetect_encoding', False)
+        self.max_tokens = kwargs.get('max_tokens', 512)
+
+    def get_content(self):
+        try:
+            if hasattr(self, 'file_path') and self.file_path:
+                with open(self.file_path, encoding=self.encoding) as f:
+                    return json.load(f)
+            elif hasattr(self, 'file_content') and self.file_content:
+                return json.load(self.file_content)
+            else:
+                raise ValueError("Neither file_path nor file_content is provided.")
+
+        except UnicodeDecodeError as e:
+            if self.autodetect_encoding:
+                if hasattr(self, 'file_path') and self.file_path:
+                    detected_encodings = detect_file_encodings(self.file_path)
+                    for encoding in detected_encodings:
+                        try:
+                            with open(self.file_path, encoding=encoding.encoding) as f:
+                                return f.read()
+                            break
+                        except UnicodeDecodeError:
+                            continue
+                elif hasattr(self, 'file_content') and self.file_content:
+                    detected_encodings = detect_file_encodings(self.file_content)
+                    for encoding in detected_encodings:
+                        try:
+                            return self.file_content.decode(encoding.encoding)
+                        except UnicodeDecodeError:
+                            continue
+                else:
+                    raise ValueError("Neither file_path nor file_content is provided for encoding detection.")
+            else:
+                raise RuntimeError(f"Error loading content with encoding {self.encoding}.") from e
+        except Exception as e:
+            raise RuntimeError(f"Error loading content.") from e
+
+    def lazy_load(self) -> Iterator[Document]:
+        """Load from file path."""
+        content_json = self.get_content()
+
+        if isinstance(content_json, list):
+            data_dict = {str(i): item for i, item in enumerate(content_json)}
+        else:
+            data_dict = content_json
+        chunks = RecursiveJsonSplitter(max_chunk_size=self.max_tokens).split_json(json_data=data_dict)
+        for chunk in chunks:
+            metadata = {"source": str(self.file_path) if hasattr(self, 'file_path') else self.file_name}
+            yield Document(page_content=json.dumps(chunk), metadata=metadata)

alita_sdk/runtime/langchain/document_loaders/constants.py

@@ -13,17 +13,18 @@
 # limitations under the License.
 
 from langchain_community.document_loaders import (
-        UnstructuredMarkdownLoader,
-        AirbyteJSONLoader, UnstructuredHTMLLoader,
-        UnstructuredPowerPointLoader, PythonLoader)
+    UnstructuredMarkdownLoader,
+    AirbyteJSONLoader, UnstructuredHTMLLoader,
+    PythonLoader)
 
 from .AlitaCSVLoader import AlitaCSVLoader
 from .AlitaDocxMammothLoader import AlitaDocxMammothLoader
 from .AlitaExcelLoader import AlitaExcelLoader
 from .AlitaImageLoader import AlitaImageLoader
+from .AlitaJSONLoader import AlitaJSONLoader
 from .AlitaPDFLoader import AlitaPDFLoader
-from .AlitaTextLoader import AlitaTextLoader
 from .AlitaPowerPointLoader import AlitaPowerPointLoader
+from .AlitaTextLoader import AlitaTextLoader
 
 loaders_map = {
     '.png': {
@@ -130,11 +131,9 @@ loaders_map = {
         'kwargs': {}
     },
     '.json': {
-        'class': AlitaTextLoader,
+        'class': AlitaJSONLoader,
         'is_multimodal_processing': False,
-        'kwargs': {
-            'autodetect_encoding': True
-        }
+        'kwargs': {}
     },
     '.jsonl': {
         'class': AirbyteJSONLoader,
@@ -154,12 +153,16 @@ loaders_map = {
     '.ppt': {
         'class': AlitaPowerPointLoader,
         'is_multimodal_processing': False,
-        'kwargs': {}
+        'kwargs': {
+            'mode': 'paged'
+        }
     },
     '.pptx': {
         'class': AlitaPowerPointLoader,
         'is_multimodal_processing': False,
-        'kwargs': {}
+        'kwargs': {
+            'mode': 'paged'
+        }
     },
     '.py': {
         'class': PythonLoader,

alita_sdk/runtime/tools/mcp_server_tool.py

@@ -1,9 +1,9 @@
 import uuid
 from logging import getLogger
-from typing import Any, Type, Literal, Optional
+from typing import Any, Type, Literal, Optional, Union, List
 
 from langchain_core.tools import BaseTool
-from pydantic import BaseModel, Field, create_model
+from pydantic import BaseModel, Field, create_model, EmailStr, constr
 
 logger = getLogger(__name__)
 
@@ -19,45 +19,73 @@ class McpServerTool(BaseTool):
 
 
     @staticmethod
-    def create_pydantic_model_from_schema(schema: dict):
-        fields = {}
-        for field_name, field_info in schema['properties'].items():
-            field_type = field_info['type']
-            field_description = field_info.get('description', '')
-            if field_type == 'string':
-                if 'enum' in field_info:
-                    field_type = Literal[tuple(field_info['enum'])]
-                else:
-                    field_type = str
-            elif field_type == 'integer':
-                field_type = int
-            elif field_type == 'number':
-                field_type = float
-            elif field_type == 'boolean':
-                field_type = bool
-            elif field_type == 'object':#Dict[str, Any]
-                nested_model = McpServerTool.create_pydantic_model_from_schema(field_info)
-                field_type = nested_model
-            elif field_type == 'array':
-                item_schema = field_info['items']
-                item_type = McpServerTool.create_pydantic_model_from_schema(item_schema) if item_schema['type'] == 'object' else (
-                    str if item_schema['type'] == 'string' else
-                    int if item_schema['type'] == 'integer' else
-                    float if item_schema['type'] == 'number' else
-                    bool if item_schema['type'] == 'boolean' else
-                    None
-                )
-                if item_type is None:
-                    raise ValueError(f"Unsupported array item type: {item_schema['type']}")
-                field_type = list[item_type]
-            else:
-                raise ValueError(f"Unsupported field type: {field_type}")
+    def create_pydantic_model_from_schema(schema: dict, model_name: str = "ArgsSchema"):
+        def parse_type(field: dict, name: str = "Field") -> Any:
+            if "allOf" in field:
+                merged = {}
+                required = set()
+                for idx, subschema in enumerate(field["allOf"]):
+                    sub_type = parse_type(subschema, f"{name}AllOf{idx}")
+                    if hasattr(sub_type, "__fields__"):
+                        merged.update({k: (v.outer_type_, v.default) for k, v in sub_type.__fields__.items()})
+                        required.update({k for k, v in sub_type.__fields__.items() if v.required})
+                if merged:
+                    return create_model(f"{name}AllOf", **merged)
+                return Any
+            if "anyOf" in field or "oneOf" in field:
+                key = "anyOf" if "anyOf" in field else "oneOf"
+                types = [parse_type(sub, f"{name}{key.capitalize()}{i}") for i, sub in enumerate(field[key])]
+                # Check for null type
+                if any(sub.get("type") == "null" for sub in field[key]):
+                    non_null_types = [parse_type(sub, f"{name}{key.capitalize()}{i}")
+                                      for i, sub in enumerate(field[key]) if sub.get("type") != "null"]
+                    if len(non_null_types) == 1:
+                        return Optional[non_null_types[0]]
+                return Union[tuple(types)]
+            t = field.get("type")
+            if isinstance(t, list):
+                if "null" in t:
+                    non_null = [x for x in t if x != "null"]
+                    if len(non_null) == 1:
+                        field = dict(field)
+                        field["type"] = non_null[0]
+                        return Optional[parse_type(field, name)]
+                    return Any
+                return Any
+            if t == "string":
+                if "enum" in field:
+                    return Literal[tuple(field["enum"])]
+                if field.get("format") == "email":
+                    return EmailStr
+                if "pattern" in field:
+                    return constr(regex=field["pattern"])
+                return str
+            if t == "integer":
+                return int
+            if t == "number":
+                return float
+            if t == "boolean":
+                return bool
+            if t == "object":
+                return McpServerTool.create_pydantic_model_from_schema(field, name.capitalize())
+            if t == "array":
+                items = field.get("items", {})
+                return List[parse_type(items, name + "Item")]
+            return Any
 
-            if field_name in schema.get('required', []):
-                fields[field_name] = (field_type, Field(..., description=field_description))
-            else:
-                fields[field_name] = (Optional[field_type], Field(None, description=field_description))
-        return create_model('DynamicModel', **fields)
+        properties = schema.get("properties", {})
+        required = set(schema.get("required", []))
+        fields = {}
+        for name, prop in properties.items():
+            typ = parse_type(prop, name.capitalize())
+            default = prop.get("default", ... if name in required else None)
+            field_args = {}
+            if "description" in prop:
+                field_args["description"] = prop["description"]
+            if "format" in prop:
+                field_args["format"] = prop["format"]
+            fields[name] = (typ, Field(default, **field_args))
+        return create_model(model_name, **fields)
 
     def _run(self, *args, **kwargs):
         call_data = {

alita_sdk/runtime/tools/vectorstore_base.py

@@ -1,18 +1,16 @@
 import json
 import math
-import types
-from typing import Any, Optional, List, Dict, Callable, Generator
+from logging import getLogger
+from typing import Any, Optional, List, Dict, Generator
 
 from langchain_core.documents import Document
-from pydantic import BaseModel, model_validator, Field
-from ..langchain.tools.vector import VectorAdapter
 from langchain_core.messages import HumanMessage
+from pydantic import BaseModel, model_validator, Field
+
 from alita_sdk.tools.elitea_base import BaseToolApiWrapper
 from alita_sdk.tools.vector_adapters.VectorStoreAdapter import VectorStoreAdapterFactory
-from logging import getLogger
-
+from ..langchain.tools.vector import VectorAdapter
 from ..utils.logging import dispatch_custom_event
-from ..utils.utils import IndexerKeywords
 
 logger = getLogger(__name__)
 
@@ -212,10 +210,6 @@ class VectorStoreWrapperBase(BaseToolApiWrapper):
             tool_name="_clean_collection"
         )
 
-    def _add_to_collection(self, entry_id, new_collection_value):
-        """Add a new collection name to the `collection` key in the `metadata` column."""
-        self.vector_adapter.add_to_collection(self, entry_id, new_collection_value)
-
     def index_documents(self, documents: Generator[Document, None, None], collection_suffix: str, progress_step: int = 20, clean_index: bool = True):
         """ Index documents in the vectorstore.
 

alita_sdk/tools/base_indexer_toolkit.py

@@ -6,7 +6,6 @@ from langchain_core.documents import Document
 from pydantic import create_model, Field, SecretStr
 
 # from alita_sdk.runtime.langchain.interfaces.llm_processor import get_embeddings
-from .chunkers import markdown_chunker
 from .utils.content_parser import process_content_by_type
 from .vector_adapters.VectorStoreAdapter import VectorStoreAdapterFactory
 from ..runtime.tools.vectorstore_base import VectorStoreWrapperBase
@@ -141,7 +140,7 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
     def _base_loader(self, **kwargs) -> Generator[Document, None, None]:
         """ Loads documents from a source, processes them,
         and returns a list of Document objects with base metadata: id and created_on."""
-        pass
+        yield from ()
 
     def _process_document(self, base_document: Document) -> Generator[Document, None, None]:
         """ Process an existing base document to extract relevant metadata for full document preparation.
@@ -153,7 +152,7 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
 
         Returns:
             Document: The processed document with metadata."""
-        pass
+        yield from ()
 
     def index_data(self, **kwargs):
         collection_suffix = kwargs.get("collection_suffix")
@@ -174,18 +173,20 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
         return self._save_index(list(documents), collection_suffix=collection_suffix, progress_step=progress_step)
     
     def _apply_loaders_chunkers(self, documents: Generator[Document, None, None], chunking_tool: str=None, chunking_config=None) -> Generator[Document, None, None]:
-        from alita_sdk.tools.chunkers import __confluence_chunkers__ as chunkers, __confluence_models__ as models
+        from alita_sdk.tools.chunkers import __confluence_chunkers__ as chunkers
 
         if chunking_config is None:
             chunking_config = {}
         chunking_config['embedding'] = self._embedding
         chunking_config['llm'] = self.llm
-            
+        
         for document in documents:
             if content_type := document.metadata.get('loader_content_type', None):
                 # apply parsing based on content type and chunk if chunker was applied to parent doc
+                content = document.metadata.pop('loader_content', None)
                 yield from process_content_by_type(
                     document=document,
+                    content=content,
                     extension_source=content_type, llm=self.llm, chunking_config=chunking_config)
             elif chunking_tool:
                 # apply default chunker from toolkit config. No parsing.
@@ -205,9 +206,6 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
             for dep in dependencies:
                 dep.metadata[IndexerKeywords.PARENT.value] = document.metadata.get('id', None)
                 yield dep
-    
-    def _content_loader(self):
-        pass
 
     def _reduce_duplicates(
             self,
@@ -255,36 +253,6 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
     def remove_ids_fn(self, idx_data, key: str):
         raise NotImplementedError("Subclasses must implement this method")
 
-    def _process_documents(self, documents: List[Document]) -> Generator[Document, None, None]:
-        """
-        Process a list of base documents to extract relevant metadata for full document preparation.
-        Used for late processing of documents after we ensure that the documents have to be indexed to avoid
-        time-consuming operations for documents which might be useless.
-        This function passed to index_documents method of vector store and called after _reduce_duplicates method.
-
-        Args:
-            documents (List[Document]): The base documents to process.
-
-        Returns:
-            Generator[Document, None, None]: A generator yielding processed documents with metadata.
-        """
-        for doc in documents:
-            # Filter documents to process only those that either:
-            # - do not have a 'chunk_id' in their metadata, or
-            # - have 'chunk_id' explicitly set to 1.
-            # This prevents processing of irrelevant or duplicate chunks, improving efficiency.
-            chunk_id = doc.metadata.get("chunk_id")
-            if chunk_id is None or chunk_id == 1:
-                processed_docs = self._process_document(doc)
-                if processed_docs:  # Only proceed if the list is not empty
-                    for processed_doc in processed_docs:
-                        # map processed document (child) to the original document (parent)
-                        processed_doc.metadata[IndexerKeywords.PARENT.value] = doc.metadata.get('id', None)
-                        if chunker:=self._get_dependencies_chunker(processed_doc):
-                            yield from chunker(file_content_generator=iter([processed_doc]), config=self._get_dependencies_chunker_config())
-                        else:
-                            yield processed_doc
-
     def remove_index(self, collection_suffix: str = ""):
         """Cleans the indexed data in the collection."""
         super()._clean_collection(collection_suffix=collection_suffix)

alita_sdk/tools/confluence/__init__.py

@@ -20,6 +20,7 @@ def get_tools(tool):
         confluence_configuration=tool['settings']['confluence_configuration'],
         limit=tool['settings'].get('limit', 5),
         labels=parse_list(tool['settings'].get('labels', None)),
+        custom_headers=tool['settings'].get('custom_headers', {}),
         additional_fields=tool['settings'].get('additional_fields', []),
         verify_ssl=tool['settings'].get('verify_ssl', True),
         alita=tool['settings'].get('alita'),
@@ -78,6 +79,8 @@ class ConfluenceToolkit(BaseToolkit):
             number_of_retries=(int, Field(description="Number of retries", default=2)),
             min_retry_seconds=(int, Field(description="Min retry, sec", default=10)),
             max_retry_seconds=(int, Field(description="Max retry, sec", default=60)),
+            # optional field for custom headers as dictionary
+            custom_headers=(Optional[dict], Field(description="Custom headers for API requests", default=None)),
             confluence_configuration=(Optional[ConfluenceConfiguration], Field(description="Confluence Configuration", json_schema_extra={'configuration_types': ['confluence']})),
             pgvector_configuration=(Optional[PgVectorConfiguration], Field(default = None,
                                                                            description="PgVector Configuration",

alita_sdk/tools/confluence/api_wrapper.py

@@ -223,16 +223,21 @@ class ConfluenceAPIWrapper(BaseVectorStoreToolApiWrapper):
         username = values.get('username')
         token = values.get('token')
         cloud = values.get('cloud')
-        # if values.get('collection_name'):
-        #     values['collection_name'] = shortuuid.encode(values['collection_name'])
         if token and is_cookie_token(token):
             session = requests.Session()
             session.cookies.update(parse_cookie_string(token))
-            values['client'] = Confluence(url=url, session=session, cloud=cloud)
+            client_instance = Confluence(url=url, session=session, cloud=cloud)
         elif token:
-            values['client'] = Confluence(url=url, token=token, cloud=cloud)
+            client_instance = Confluence(url=url, token=token, cloud=cloud)
         else:
-            values['client'] = Confluence(url=url, username=username, password=api_key, cloud=cloud)
+            client_instance = Confluence(url=url, username=username, password=api_key, cloud=cloud)
+
+        custom_headers = values.get('custom_headers', {})
+        logger.info(f"Jira tool: custom headers length: {len(custom_headers)}")
+        for header, value in custom_headers.items():
+            client_instance._update_header(header, value)
+
+        values['client'] = client_instance
         return values
 
     def __unquote_confluence_space(self) -> str | None:

alita_sdk/tools/elitea_base.py

@@ -124,6 +124,26 @@ class BaseToolApiWrapper(BaseModel):
     def get_available_tools(self):
         raise NotImplementedError("Subclasses should implement this method")
 
+    def _log_tool_event(self, message: str, tool_name: str = None):
+        """Log data and dispatch custom event for the tool"""
+
+        try:
+            from langchain_core.callbacks import dispatch_custom_event
+
+            if tool_name is None:
+                tool_name = 'tool_progress'
+            dispatch_custom_event(
+                name="tool_execution_step",
+                data={
+                    "message": message,
+                    "tool_name": tool_name,
+                    "toolkit": self.__class__.__name__,
+                },
+            )
+        except Exception as e:
+            logger.warning(f"Failed to dispatch progress event: {str(e)}")
+
+
     def run(self, mode: str, *args: Any, **kwargs: Any):
         if TOOLKIT_SPLITTER in mode:
             mode = mode.rsplit(TOOLKIT_SPLITTER, maxsplit=1)[1]

alita_sdk/tools/sharepoint/api_wrapper.py

@@ -1,6 +1,5 @@
-import json
 import logging
-from typing import Optional, List, Generator, Any
+from typing import Optional, Generator
 
 from langchain_core.documents import Document
 from langchain_core.tools import ToolException
@@ -8,7 +7,7 @@ from office365.runtime.auth.client_credential import ClientCredential
 from office365.sharepoint.client_context import ClientContext
 from pydantic import Field, PrivateAttr, create_model, model_validator, SecretStr
 
-from ..elitea_base import BaseVectorStoreToolApiWrapper, extend_with_vector_tools
+from ..non_code_indexer_toolkit import NonCodeIndexerToolkit
 from ..utils.content_parser import parse_file_content
 
 NoInput = create_model(
@@ -38,7 +37,7 @@ ReadDocument = create_model(
 )
 
 
-class SharepointApiWrapper(BaseVectorStoreToolApiWrapper):
+class SharepointApiWrapper(NonCodeIndexerToolkit):
     site_url: str
     client_id: str = None
     client_secret: SecretStr = None
@@ -77,9 +76,8 @@ class SharepointApiWrapper(BaseVectorStoreToolApiWrapper):
                 raise ToolException("You have to define token or client id&secret.")
             logging.info("Successfully authenticated to SharePoint.")
         except Exception as e:
-                logging.error(f"Failed to authenticate with SharePoint: {str(e)}")
-        return values
-
+            logging.error(f"Failed to authenticate with SharePoint: {str(e)}")
+        return super().validate_toolkit(values)
 
     def read_list(self, list_title, limit: int = 1000):
         """ Reads a specified List in sharepoint site. Number of list items is limited by limit (default is 1000). """
@@ -161,25 +159,26 @@ class SharepointApiWrapper(BaseVectorStoreToolApiWrapper):
             }
             yield Document(page_content="", metadata=metadata)
 
-    def _process_document(self, document: Document) -> Generator[Document, None, None]:
-        doc_content = ""
-        try:
-            doc_content = self.read_file(document.metadata['Path'],
-                                      is_capture_image=True,
-                                      excel_by_sheets=True)
-        except Exception as e:
-            logging.error(f"Failed while parsing the file '{document.metadata['Path']}': {e}")
-        if isinstance(doc_content, dict):
-            for page, content in doc_content.items():
-                new_metadata = document.metadata
-                new_metadata['page'] = page
-                yield Document(page_content=str(content), metadata=new_metadata)
-        else:
-            document.page_content = str(doc_content)
-
-    @extend_with_vector_tools
+    def _extend_data(self, documents: Generator[Document, None, None]):
+        for document in documents:
+            try:
+                document.metadata['loader_content'] = self._load_file_content_in_bytes(document.metadata['Path'])
+                document.metadata['loader_content_type'] = document.metadata['Name']
+                yield document
+            except Exception as e:
+                logging.error(f"Failed while parsing the file '{document.metadata['Path']}': {e}")
+                yield document
+
+    def _load_file_content_in_bytes(self, path):
+        file = self._client.web.get_file_by_server_relative_path(path)
+        self._client.load(file).execute_query()
+        file_content = file.read()
+        self._client.execute_query()
+        #
+        return file_content
+
     def get_available_tools(self):
-        return [
+        return super().get_available_tools() + [
             {
                 "name": "read_list",
                 "description": self.read_list.__doc__,

alita_sdk/tools/testrail/api_wrapper.py

@@ -466,11 +466,11 @@ class TestrailAPIWrapper(BaseVectorStoreToolApiWrapper):
                 return ToolException(
                     "json_case_arguments must be a JSON string or dictionary."
                 )
-
+            self._log_tool_event(message=f"Extract test cases per filter {params}", tool_name='get_cases_by_filter')
             extracted_cases = self._client.cases.get_cases(
                 project_id=project_id, **params
             )
-
+            self._log_tool_event(message=f"Test cases were extracted", tool_name='get_cases_by_filter')
             # support old versions of testrail_api
             cases = extracted_cases.get("cases") if isinstance(extracted_cases, dict) else extracted_cases
 

alita_sdk/tools/utils/content_parser.py

@@ -170,14 +170,17 @@ def load_content_from_bytes(file_content: bytes, extension: str = None, loader_e
         if temp_file_path and os.path.exists(temp_file_path):
             os.remove(temp_file_path)
 
-def process_content_by_type(document: Document, extension_source: str, llm = None, chunking_config={}) -> Generator[Document, None, None]:
+def process_content_by_type(document: Document, content, extension_source: str, llm = None, chunking_config={}) -> Generator[Document, None, None]:
     temp_file_path = None
     try:
         extension = "." + extension_source.split('.')[-1].lower()
 
         with tempfile.NamedTemporaryFile(mode='w+b', suffix=extension, delete=False) as temp_file:
             temp_file_path = temp_file.name
-            content = document.metadata.pop('loader_content')
+            if content is None:
+                logger.warning("'loader_content' ie expected but not found in document metadata.")
+                return
+            
             temp_file.write(content)
             temp_file.flush()
 
@@ -207,7 +210,7 @@ def process_content_by_type(document: Document, extension_source: str, llm = Non
                         docmeta['chunk_id'] = chunk_id
                         docmeta['chunk_type'] = "document"
                         yield Document(
-                            page_content=subchunk,
+                            page_content=sanitize_for_postgres(subchunk),
                             metadata=docmeta
                         )
                 else:
@@ -218,9 +221,30 @@ def process_content_by_type(document: Document, extension_source: str, llm = Non
                     docmeta['chunk_id'] = chunk_id
                     docmeta['chunk_type'] = "document"
                     yield Document(
-                        page_content=chunk.page_content,
+                        page_content=sanitize_for_postgres(chunk.page_content),
                         metadata=docmeta
                     )
     finally:
         if temp_file_path and os.path.exists(temp_file_path):
-            os.remove(temp_file_path)
+            os.remove(temp_file_path)
+
+# FIXME copied from langchain_core/utils/strings.py of 0.3.74 version
+# https://github.com/langchain-ai/langchain/pull/32157
+# should be used from langchain_core.utils import sanitize_for_postgres once updated to newer version
+def sanitize_for_postgres(text: str, replacement: str = "") -> str:
+    r"""Sanitize text by removing NUL bytes that are incompatible with PostgreSQL.
+    PostgreSQL text fields cannot contain NUL (0x00) bytes, which can cause
+    psycopg.DataError when inserting documents. This function removes or replaces
+    such characters to ensure compatibility.
+    Args:
+        text: The text to sanitize.
+        replacement: String to replace NUL bytes with. Defaults to empty string.
+    Returns:
+        str: The sanitized text with NUL bytes removed or replaced.
+    Example:
+        >>> sanitize_for_postgres("Hello\\x00world")
+        'Helloworld'
+        >>> sanitize_for_postgres("Hello\\x00world", " ")
+        'Hello world'
+    """
+    return text.replace("\x00", replacement)

{alita_sdk-0.3.253.dist-info → alita_sdk-0.3.254.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.253
+Version: 0.3.254
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedjik@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.253.dist-info → alita_sdk-0.3.254.dist-info}/RECORD

@@ -26,7 +26,7 @@ alita_sdk/configurations/zephyr_enterprise.py,sha256=5W1QEcv62Y5Rk_kApI2QmOwvWZe
 alita_sdk/runtime/__init__.py,sha256=4W0UF-nl3QF2bvET5lnah4o24CoTwSoKXhuN0YnwvEE,828
 alita_sdk/runtime/clients/__init__.py,sha256=BdehU5GBztN1Qi1Wul0cqlU46FxUfMnI6Vq2Zd_oq1M,296
 alita_sdk/runtime/clients/artifact.py,sha256=H3pJAh5G-zWVyJ6YbqHGk4jA8U6HfacQduiTivpJZ3Y,3210
-alita_sdk/runtime/clients/client.py,sha256=HO5mSrrque9HaHdBmQVR639leBNTI1TFA0HYmXdfqLA,43187
+alita_sdk/runtime/clients/client.py,sha256=irj2uTGdIQj8Wd1ZGdi5yDCFm_n9TiRhEhODJz4yI84,43493
 alita_sdk/runtime/clients/datasource.py,sha256=HAZovoQN9jBg0_-lIlGBQzb4FJdczPhkHehAiVG3Wx0,1020
 alita_sdk/runtime/clients/prompt.py,sha256=li1RG9eBwgNK_Qf0qUaZ8QNTmsncFrAL2pv3kbxZRZg,1447
 alita_sdk/runtime/langchain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -49,6 +49,7 @@ alita_sdk/runtime/langchain/document_loaders/AlitaDocxMammothLoader.py,sha256=nH
 alita_sdk/runtime/langchain/document_loaders/AlitaExcelLoader.py,sha256=YBFYikrOEITfIavU0Xu7BQSNvPCFKzcmbJ_VDeQ6KdI,3078
 alita_sdk/runtime/langchain/document_loaders/AlitaGitRepoLoader.py,sha256=5WXGcyHraSVj3ANHj_U6X4EDikoekrIYtS0Q_QqNIng,2608
 alita_sdk/runtime/langchain/document_loaders/AlitaImageLoader.py,sha256=ogvCmpnS54-D7fP_sSkL1dnhHTmRSD-HA2FFrTNhDEo,6560
+alita_sdk/runtime/langchain/document_loaders/AlitaJSONLoader.py,sha256=1mGZjltnqsSXkp1Jw-lQroyNFiCPpjb9ZbdoqOlqPeU,3354
 alita_sdk/runtime/langchain/document_loaders/AlitaJiraLoader.py,sha256=M2q3YThkps0yAZOjfoLcyE7qycVTYKcXEGtpmp0N6C8,10950
 alita_sdk/runtime/langchain/document_loaders/AlitaPDFLoader.py,sha256=elymFlVGiCkcrIY5FrLxbxnQ9jdt3PPV0yBJGF3pTFE,2858
 alita_sdk/runtime/langchain/document_loaders/AlitaPowerPointLoader.py,sha256=SKAAPo3DfMtRPxICKrPzlXXkC5RfaeiRj7lejLXTi7o,2337
@@ -56,7 +57,7 @@ alita_sdk/runtime/langchain/document_loaders/AlitaQtestLoader.py,sha256=CUVVnisx
 alita_sdk/runtime/langchain/document_loaders/AlitaTableLoader.py,sha256=o0SRFPZ-VskltgThVRX80rT19qtB4gPzxED9SENTNWo,4145
 alita_sdk/runtime/langchain/document_loaders/AlitaTextLoader.py,sha256=uNcV0En49_0u0RYB1sP1XfNspT2Xc5CacuJr9Jqv79Q,2972
 alita_sdk/runtime/langchain/document_loaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-alita_sdk/runtime/langchain/document_loaders/constants.py,sha256=Zu_TSxZzcsrJjLEfLSgPEmoZOn97hwI-lfKJR0JKJzk,4535
+alita_sdk/runtime/langchain/document_loaders/constants.py,sha256=gTTHIbJQVpSGaOwQjJwAltZryoDDX7GaqbODI30MwQM,4563
 alita_sdk/runtime/langchain/document_loaders/utils.py,sha256=9xghESf3axBbwxATyVuS0Yu-TWe8zWZnXgCD1ZVyNW0,2414
 alita_sdk/runtime/langchain/interfaces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 alita_sdk/runtime/langchain/interfaces/kwextractor.py,sha256=kSJA9L8g8UArmHu7Bd9dIO0Rrq86JPUb8RYNlnN68FQ,3072
@@ -100,13 +101,13 @@ alita_sdk/runtime/tools/indexer_tool.py,sha256=whSLPevB4WD6dhh2JDXEivDmTvbjiMV1M
 alita_sdk/runtime/tools/llm.py,sha256=NsrsP-SblyxDdzgMCn9_OBUL0sUGDVS5yqer49V7ciE,15069
 alita_sdk/runtime/tools/loop.py,sha256=uds0WhZvwMxDVFI6MZHrcmMle637cQfBNg682iLxoJA,8335
 alita_sdk/runtime/tools/loop_output.py,sha256=U4hO9PCQgWlXwOq6jdmCGbegtAxGAPXObSxZQ3z38uk,8069
-alita_sdk/runtime/tools/mcp_server_tool.py,sha256=eI8QUt497xblwF4Zhbvi8wCg17yh2yoWjcw_AIzHwGE,2819
+alita_sdk/runtime/tools/mcp_server_tool.py,sha256=trGraI8-AwdbNmTKMjfmlBxgTDMTE4-21heCVtd_lz0,4156
 alita_sdk/runtime/tools/pgvector_search.py,sha256=NN2BGAnq4SsDHIhUcFZ8d_dbEOM8QwB0UwpsWCYruXU,11692
 alita_sdk/runtime/tools/prompt.py,sha256=nJafb_e5aOM1Rr3qGFCR-SKziU9uCsiP2okIMs9PppM,741
 alita_sdk/runtime/tools/router.py,sha256=wCvZjVkdXK9dMMeEerrgKf5M790RudH68pDortnHSz0,1517
 alita_sdk/runtime/tools/tool.py,sha256=lE1hGi6qOAXG7qxtqxarD_XMQqTghdywf261DZawwno,5631
 alita_sdk/runtime/tools/vectorstore.py,sha256=l5wfovwMNvS_RgW-ZHXCh8Cm8gauunRzP0NPkzmshcQ,33852
-alita_sdk/runtime/tools/vectorstore_base.py,sha256=OdJIJkjTmQ0BC-AzAOMP2phAcNATJ8gI5JoBWSSdpNU,27892
+alita_sdk/runtime/tools/vectorstore_base.py,sha256=HFaNk_oBoeZWrQWBrvEsozajHqwjWxsV6RigkQyq-eQ,27586
 alita_sdk/runtime/utils/AlitaCallback.py,sha256=E4LlSBuCHWiUq6W7IZExERHZY0qcmdjzc_rJlF2iQIw,7356
 alita_sdk/runtime/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 alita_sdk/runtime/utils/constants.py,sha256=Xntx1b_uxUzT4clwqHA_U6K8y5bBqf_4lSQwXdcWrp4,13586
@@ -118,8 +119,8 @@ alita_sdk/runtime/utils/toolkit_runtime.py,sha256=MU63Fpxj0b5_r1IUUc0Q3-PN9VwL7r
 alita_sdk/runtime/utils/toolkit_utils.py,sha256=I9QFqnaqfVgN26LUr6s3XlBlG6y0CoHURnCzG7XcwVs,5311
 alita_sdk/runtime/utils/utils.py,sha256=CpEl3LCeLbhzQySz08lkKPm7Auac6IiLF7WB8wmArMI,589
 alita_sdk/tools/__init__.py,sha256=ko5TToGYZFmBrho26DRAVvrkHWxQ2sfs8gVAASinYp8,10611
-alita_sdk/tools/base_indexer_toolkit.py,sha256=qQfMHzsQ2BfusKMV_DNiHOtZVheiQ4gBfy5JXjYi0UY,20231
-alita_sdk/tools/elitea_base.py,sha256=qXSrl0A8KxIuv6796bTkjPpxBm4WQ5zmpskIAwCFfC8,30394
+alita_sdk/tools/base_indexer_toolkit.py,sha256=gOjE1igKyjG1LohMj0XMlj1IGaFp7eEEDqyEG6-xLmc,18405
+alita_sdk/tools/elitea_base.py,sha256=yfDSNKdLtsedp8546KHfDPOPYgpD4ZZ-dvAxs3zoF3o,31071
 alita_sdk/tools/non_code_indexer_toolkit.py,sha256=v9uq1POE1fQKCd152mbqDtF-HSe0qoDj83k4E5LAkMI,1080
 alita_sdk/tools/ado/__init__.py,sha256=bArTObt5cqG1SkijKevWGbsIILHBA3aCStg8Q1jd69k,1243
 alita_sdk/tools/ado/utils.py,sha256=PTCludvaQmPLakF2EbCGy66Mro4-rjDtavVP-xcB2Wc,1252
@@ -211,8 +212,8 @@ alita_sdk/tools/code/linter/api_wrapper.py,sha256=wylpwhAw02Jt8L18CqBq2He5PbwIkx
 alita_sdk/tools/code/loaders/codesearcher.py,sha256=XoXXZtIQZhvjIwZlnl_4wVGHC-3saYzFo5oDR_Zh3EY,529
 alita_sdk/tools/code/sonar/__init__.py,sha256=u8wpgXJ_shToLl3G9-XEtGDor5dhmsnurIImh1-e-U0,3165
 alita_sdk/tools/code/sonar/api_wrapper.py,sha256=nNqxcWN_6W8c0ckj-Er9HkNuAdgQLoWBXh5UyzNutis,2653
-alita_sdk/tools/confluence/__init__.py,sha256=xLsxdBZ62NL0k9NxaV4KnspwmDcucQzcl-tAaz7eLB8,6562
-alita_sdk/tools/confluence/api_wrapper.py,sha256=4WqjVeFWyFeb4-VD5v4_J69pbyjire4Op7cBSKU9EXw,85057
+alita_sdk/tools/confluence/__init__.py,sha256=ClK6fuJr5rsgDhLLA2Ci3zJdBSP3liyUpHML9oQqKFs,6804
+alita_sdk/tools/confluence/api_wrapper.py,sha256=-wQduJUk2wwfBQGQHSWuCnrt35gfp195nSKUFVSPS1s,85218
 alita_sdk/tools/confluence/loader.py,sha256=4bf5qrJMEiJzuZp2NlxO2XObLD1w7fxss_WyMUpe8sg,9290
 alita_sdk/tools/confluence/utils.py,sha256=Lxo6dBD0OlvM4o0JuK6qeB_4LV9BptiwJA9e1vqNcDw,435
 alita_sdk/tools/custom_open_api/__init__.py,sha256=9aT5SPNPWcJC6jMZEM-3rUCXVULj_3-qJLQKmnreKNo,2537
@@ -298,7 +299,7 @@ alita_sdk/tools/servicenow/__init__.py,sha256=hReiTp8yv07eR0O_1KJThzUO2xhWhIWcjU
 alita_sdk/tools/servicenow/api_wrapper.py,sha256=WpH-bBLGFdhehs4g-K-WAkNuaD1CSrwsDpdgB3RG53s,6120
 alita_sdk/tools/servicenow/servicenow_client.py,sha256=Rdqfu-ll-qbnclMzChLZBsfXRDzgoX_FdeI2WLApWxc,3269
 alita_sdk/tools/sharepoint/__init__.py,sha256=Mofg_N-7zFf5mKm3_0D0dhC_H0MX-bk3YQ5Sl3oXokg,4114
-alita_sdk/tools/sharepoint/api_wrapper.py,sha256=TSdKZuLnn3uSkaNuYb7a2xG4w4sQzXbzOO3c8tIlFds,9259
+alita_sdk/tools/sharepoint/api_wrapper.py,sha256=Hcd9YypWMr3upDVJHRxUyPdN4k8joqRQOc_uce2ek1A,9250
 alita_sdk/tools/sharepoint/authorization_helper.py,sha256=n-nL5dlBoLMK70nHu7P2RYCb8C6c9HMA_gEaw8LxuhE,2007
 alita_sdk/tools/sharepoint/utils.py,sha256=fZ1YzAu5CTjKSZeslowpOPH974902S8vCp1Wu7L44LM,446
 alita_sdk/tools/slack/__init__.py,sha256=o8BnDMWGC5qA8pVIyIiflM6T__dZ6qAE1UdtJcvmaxk,3901
@@ -309,9 +310,9 @@ alita_sdk/tools/sql/models.py,sha256=AKJgSl_kEEz4fZfw3kbvdGHXaRZ-yiaqfJOB6YOj3i0
 alita_sdk/tools/testio/__init__.py,sha256=qi12wyJXN02hrUXg08CbijcCL5pi30JMbJfiXjn1Zr0,2646
 alita_sdk/tools/testio/api_wrapper.py,sha256=BvmL5h634BzG6p7ajnQLmj-uoAw1gjWnd4FHHu1h--Q,21638
 alita_sdk/tools/testrail/__init__.py,sha256=0kETjWKLU7R6mugBWsjwEUsh10pipbAeNSGJAO0FBh0,4634
-alita_sdk/tools/testrail/api_wrapper.py,sha256=K-Gc42RH2z-fK4cXi8zQq3s9A4v_pCJkRB3XKLAhypc,32056
+alita_sdk/tools/testrail/api_wrapper.py,sha256=5T-QyTzt-J0rI32xc_E684lCdgyWeHSyeTYiwQwtGyg,32275
 alita_sdk/tools/utils/__init__.py,sha256=155xepXPr4OEzs2Mz5YnjXcBpxSv1X2eznRUVoPtyK0,3268
-alita_sdk/tools/utils/content_parser.py,sha256=0HKQqGTdXHKlcz72GHEwXqLXJsRYXm35F-P1KZz0sNc,10351
+alita_sdk/tools/utils/content_parser.py,sha256=MsBlh97v5aVTuB2bw43J4K2-IJumKOoRbz0zkpjkbhI,11521
 alita_sdk/tools/vector_adapters/VectorStoreAdapter.py,sha256=a6FAsiix_EvATIKUf5YT6vHh5LDyJ5uSP3LJqoxFo04,17367
 alita_sdk/tools/vector_adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 alita_sdk/tools/xray/__init__.py,sha256=GGpbiBdDQ9kMFqJEHYi7XwKpkuMMHi-ZF-IM8yFIgUM,4380
@@ -333,8 +334,8 @@ alita_sdk/tools/zephyr_scale/api_wrapper.py,sha256=JAeWf-RXohsxheUpT0iMDClc_izj-
 alita_sdk/tools/zephyr_squad/__init__.py,sha256=0AI_j27xVO5Gk5HQMFrqPTd4uvuVTpiZUicBrdfEpKg,2796
 alita_sdk/tools/zephyr_squad/api_wrapper.py,sha256=kmw_xol8YIYFplBLWTqP_VKPRhL_1ItDD0_vXTe_UuI,14906
 alita_sdk/tools/zephyr_squad/zephyr_squad_cloud_client.py,sha256=R371waHsms4sllHCbijKYs90C-9Yu0sSR3N4SUfQOgU,5066
-alita_sdk-0.3.253.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-alita_sdk-0.3.253.dist-info/METADATA,sha256=sOv_LdDPyuyBm4c-1hfZH1XG_V5-MeUIDuTJgmDX8Hk,18897
-alita_sdk-0.3.253.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-alita_sdk-0.3.253.dist-info/top_level.txt,sha256=0vJYy5p_jK6AwVb1aqXr7Kgqgk3WDtQ6t5C-XI9zkmg,10
-alita_sdk-0.3.253.dist-info/RECORD,,
+alita_sdk-0.3.254.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+alita_sdk-0.3.254.dist-info/METADATA,sha256=XLAiAMkY0U1LualoozT6xq8wieW8dvFe0xwIZeM7Dmw,18897
+alita_sdk-0.3.254.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+alita_sdk-0.3.254.dist-info/top_level.txt,sha256=0vJYy5p_jK6AwVb1aqXr7Kgqgk3WDtQ6t5C-XI9zkmg,10
+alita_sdk-0.3.254.dist-info/RECORD,,