alita-sdk 0.3.249__py3-none-any.whl → 0.3.250__py3-none-any.whl

alita_sdk/runtime/tools/vectorstore_base.py

@@ -0,0 +1,624 @@
+import json
+import math
+import types
+from typing import Any, Optional, List, Dict, Callable, Generator
+
+from langchain_core.documents import Document
+from pydantic import BaseModel, model_validator, Field
+from ..langchain.tools.vector import VectorAdapter
+from langchain_core.messages import HumanMessage
+from alita_sdk.tools.elitea_base import BaseToolApiWrapper
+from alita_sdk.tools.vector_adapters.VectorStoreAdapter import VectorStoreAdapterFactory
+from logging import getLogger
+
+from ..utils.logging import dispatch_custom_event
+from ..utils.utils import IndexerKeywords
+
+logger = getLogger(__name__)
+
+class IndexDocumentsModel(BaseModel):
+    documents: Any = Field(description="Generator of documents to index")
+
+class SearchDocumentsModel(BaseModel):
+    query: str = Field(description="Search query")
+    doctype: str = Field(description="Document type")
+    filter: Optional[dict | str] = Field(
+        description='Filter for metadata of documents. Use JSON format for complex filters.',
+        default=None)
+    search_top: Optional[int] = Field(description="Number of search results", default=10)
+    cut_off: Optional[float] = Field(description="Cut off value for search results", default=0.5)
+    full_text_search: Optional[Dict[str, Any]] = Field(
+        description="""Full text search configuration. Example:
+        {
+            "enabled": true,
+            "weight": 0.3,
+            "fields": ["content", "title"],
+            "language": "english"
+        }""",
+        default=None
+    )
+    reranking_config: Optional[Dict[str, Dict[str, Any]]] = Field(
+        description="""Reranking configuration. Example:
+        {
+            "field_name": {
+                "weight": 1.0,
+                "rules": {
+                    "contains": "keyword",
+                    "priority": "value",
+                    "sort": "desc"
+                }
+            }
+        }""",
+        default=None
+    )
+    extended_search: Optional[List[str]] = Field(
+        description="List of chunk types to search for (title, summary, propositions, keywords, documents)",
+        default=None
+    )
+
+class StepBackSearchDocumentsModel(BaseModel):
+    query: str = Field(description="Search query")
+    doctype: str = Field(description="Document type")
+    messages: Optional[list] = Field(description="Conversation history", default=[])
+    filter: Optional[dict] = Field(description='Filter for metadata of documents. Use JSON format for complex filters.', default=None)
+    search_top: Optional[int] = Field(description="Number of search results", default=10)
+    cut_off: Optional[float] = Field(description="Cut off value for search results", default=0.5)
+    full_text_search: Optional[Dict[str, Any]] = Field(
+        description="""Full text search configuration. Example:
+        {
+            "enabled": true,
+            "weight": 0.3,
+            "fields": ["content", "title"],
+            "language": "english"
+        }""",
+        default=None
+    )
+    reranking_config: Optional[Dict[str, Dict[str, Any]]] = Field(
+        description="""Reranking configuration. Example:
+        {
+            "field_name": {
+                "weight": 1.0,
+                "rules": {
+                    "contains": "keyword",
+                    "priority": "value",
+                    "sort": "desc"
+                }
+            }
+        }""",
+        default=None
+    )
+    extended_search: Optional[List[str]] = Field(
+        description="List of chunk types to search for (title, summary, propositions, keywords, documents)",
+        default=None
+    )
+
+STEPBACK_PROMPT = """Your task is to convert provided question into a more generic question that will be used for similarity search.
+Remove all not important words, question words, but save all names, dates and acronym as in original question.
+
+<input>
+{input} 
+</input>
+
+Output:
+"""
+
+GET_ANSWER_PROMPT = """<search_results>
+{search_results}
+</search_results>
+
+<conversation_history>
+{messages}
+</conversation_history>
+
+Please answer the question based on provided search results.
+Provided information is already processed and available in the context as list of possibly relevant pieces of the documents.
+Use only provided information. Do not make up answer.
+If you have no answer and you can not derive it from the context, please provide "I have no answer".
+<question>
+{input}
+</question>
+## Answer
+Add <ANSWER> here
+
+## Score
+Score the answer from 0 to 100, where 0 is not relevant and 100 is very relevant.
+
+## Citations
+- source (score)
+- source (score)
+Make sure to provide unique source for each citation.
+
+## Explanation
+How did you come up with the answer?
+"""
+
+class VectorStoreWrapperBase(BaseToolApiWrapper):
+    llm: Any
+    embedding_model: str
+    embedding_model_params: dict
+    vectorstore_type: str
+    vectorstore_params: dict
+    max_docs_per_add: int = 100
+    dataset: str = None
+    embedding: Any = None
+    vectorstore: Any = None
+    # Review usage of old adapter
+    vectoradapter: Any = None
+    pg_helper: Any = None
+    embeddings: Any = None
+    # New adapter for vector database operations
+    vector_adapter: Any = None
+
+    @model_validator(mode='before')
+    @classmethod
+    def validate_toolkit(cls, values):
+        from ..langchain.interfaces.llm_processor import get_embeddings, get_vectorstore
+        logger.debug(f"Validating toolkit: {values}")
+        if not values.get('vectorstore_type'):
+            raise ValueError("Vectorstore type is required.")
+        if not values.get('embedding_model'):
+            raise ValueError("Embedding model is required.")
+        if not values.get('vectorstore_params'):
+            raise ValueError("Vectorstore parameters are required.")
+        if not values.get('embedding_model_params'):
+            raise ValueError("Embedding model parameters are required.")
+        values["dataset"] = values.get('vectorstore_params').get('collection_name')
+        if not values["dataset"]:
+            raise ValueError("Collection name is required.")
+        if not values.get('embeddings'):
+            values['embeddings'] = get_embeddings(values['embedding_model'], values['embedding_model_params'])
+        values['vectorstore'] = get_vectorstore(values['vectorstore_type'], values['vectorstore_params'], embedding_func=values['embeddings'])
+        values['vectoradapter'] = VectorAdapter(
+            vectorstore=values['vectorstore'],
+            embeddings=values['embeddings'],
+            quota_params=None,
+        )
+        # Initialize the new vector adapter
+        values['vector_adapter'] = VectorStoreAdapterFactory.create_adapter(values['vectorstore_type'])
+        logger.debug(f"Vectorstore wrapper initialized: {values}")
+        return values
+
+    def _init_pg_helper(self, language='english'):
+        """Initialize PGVector helper if needed and not already initialized"""
+        if self.pg_helper is None and hasattr(self.vectorstore, 'connection_string') and hasattr(self.vectorstore, 'collection_name'):
+            try:
+                from .pgvector_search import PGVectorSearch
+                self.pg_helper = PGVectorSearch(
+                    self.vectorstore.connection_string,
+                    self.vectorstore.collection_name,
+                    language=language
+                )
+            except ImportError:
+                logger.warning("PGVectorSearch not available - full-text search will be limited")
+            except Exception as e:
+                logger.error(f"Failed to initialize PGVectorSearch: {str(e)}")
+
+    def list_collections(self) -> List[str]:
+        """List all collections in the vectorstore."""
+
+        return self.vector_adapter.list_collections(self)
+
+    def _clean_collection(self, collection_suffix: str = ''):
+        """
+        Clean the vectorstore collection by deleting all indexed data.
+        """
+        self._log_data(
+            f"Cleaning collection '{self.dataset}'",
+            tool_name="_clean_collection"
+        )
+        self.vector_adapter.clean_collection(self, collection_suffix)
+        self._log_data(
+            f"Collection '{self.dataset}' has been cleaned. ",
+            tool_name="_clean_collection"
+        )
+
+    def _add_to_collection(self, entry_id, new_collection_value):
+        """Add a new collection name to the `collection` key in the `metadata` column."""
+        self.vector_adapter.add_to_collection(self, entry_id, new_collection_value)
+
+    def index_documents(self, documents: Generator[Document, None, None], collection_suffix: str, progress_step: int = 20, clean_index: bool = True):
+        """ Index documents in the vectorstore.
+
+        Args:
+            documents (Any): Generator or list of documents to index.
+            progress_step (int): Step for progress reporting, default is 20.
+            clean_index (bool): If True, clean the index before re-indexing all documents.
+        """
+        if clean_index:
+            self._clean_index(collection_suffix)
+
+        return self._save_index(list(documents), collection_suffix, progress_step)
+
+    def _clean_index(self, collection_suffix: str):
+        logger.info("Cleaning index before re-indexing all documents.")
+        self._log_data("Cleaning index before re-indexing all documents. Previous index will be removed", tool_name="index_documents")
+        try:
+            self._clean_collection(collection_suffix)
+            self.vectoradapter.persist()
+            self.vectoradapter.vacuum()
+            self._log_data("Previous index has been removed",
+                           tool_name="index_documents")
+        except Exception as e:
+            logger.warning(f"Failed to clean index: {str(e)}. Continuing with re-indexing.")
+
+    def _save_index(self, documents: list[Document], collection_suffix: Optional[str] = None, progress_step: int = 20):
+        from ..langchain.interfaces.llm_processor import add_documents
+        #
+        for doc in documents:
+            if 'id' not in doc.metadata or 'updated_on' not in doc.metadata:
+                logger.warning(f"Document is missing required metadata field 'id' or 'updated_on': {doc.metadata}")
+
+        logger.debug(f"Indexing documents: {documents}")
+        logger.debug(self.vectoradapter)
+
+        # if collection_suffix is provided, add it to metadata of each document
+        if collection_suffix:
+            for doc in documents:
+                if not doc.metadata.get('collection'):
+                    doc.metadata['collection'] = collection_suffix
+                else:
+                    doc.metadata['collection'] += f";{collection_suffix}"
+
+        total_docs = len(documents)
+        documents_count = 0
+        _documents = []
+
+        # set default progress step to 20 if out of 0...100 or None
+        progress_step = 20 if progress_step not in range(0, 100) else progress_step
+        next_progress_point = progress_step
+        for document in documents:
+            documents_count += 1
+            # logger.debug(f"Indexing document: {document}")
+            try:
+                _documents.append(document)
+                if len(_documents) >= self.max_docs_per_add:
+                    add_documents(vectorstore=self.vectorstore, documents=_documents)
+                    _documents = []
+
+                percent = math.floor((documents_count / total_docs) * 100)
+                if percent >= next_progress_point:
+                    msg = f"Indexing progress: {percent}%. Processed {documents_count} of {total_docs} documents."
+                    logger.debug(msg)
+                    self._log_data(msg)
+                    next_progress_point += progress_step
+            except Exception:
+                from traceback import format_exc
+                logger.error(f"Error: {format_exc()}")
+                return {"status": "error", "message": f"Error: {format_exc()}"}
+        if _documents:
+            add_documents(vectorstore=self.vectorstore, documents=_documents)
+        return {"status": "ok", "message": f"successfully indexed {documents_count} documents"}
+
+    def search_documents(self, query:str, doctype: str = 'code', 
+                         filter:dict|str={}, cut_off: float=0.5,
+                         search_top:int=10, full_text_search: Optional[Dict[str, Any]] = None,
+                         extended_search: Optional[List[str]] = None,
+                         reranker: dict = {}, reranking_config: Optional[Dict[str, Dict[str, Any]]] = None
+                         ):
+        """Enhanced search documents method using JSON configurations for full-text search and reranking"""
+        from alita_sdk.tools.code.loaders.codesearcher import search_format as code_format
+        
+        if not filter:
+            filter = None
+        else:
+            if isinstance(filter, str):
+                filter = json.loads(filter)
+
+        # Extended search implementation
+        if extended_search:
+            # Track unique documents by source and chunk_id
+            unique_docs = {}
+            chunk_type_scores = {}  # Store scores by document identifier
+            # Create initial set of results from documents
+            if filter is None:
+                document_filter = {"chunk_type": {"$eq": "document"}}
+            else:
+                document_filter = {
+                    "$and": [
+                        filter,
+                        {"chunk_type": {"$eq": "document"}}
+                    ]
+                }
+                
+            try:
+                document_items = self.vectorstore.similarity_search_with_score(
+                    query, filter=document_filter, k=search_top
+                )                
+                # Add document results to unique docs
+                vector_items = document_items
+                for doc, score in document_items:
+                    source = doc.metadata.get('source')
+                    chunk_id = doc.metadata.get('chunk_id')
+                    doc_id = f"{source}_{chunk_id}" if source and chunk_id else str(doc.metadata.get('id', id(doc)))
+                    
+                    if doc_id not in unique_docs or score > chunk_type_scores.get(doc_id, 0):
+                        unique_docs[doc_id] = doc
+                        chunk_type_scores[doc_id] = score
+            except Exception as e:
+                logger.warning(f"Error searching for document chunks: {str(e)}")
+            
+            # First search for specified chunk types (title, summary, propositions, keywords)
+            valid_chunk_types = ["title", "summary", "propositions", "keywords"]
+            chunk_types_to_search = [ct for ct in extended_search if ct in valid_chunk_types]
+            
+            # Search for each chunk type separately
+            for chunk_type in chunk_types_to_search:
+                if filter is None:
+                    chunk_filter = {"chunk_type": {"$eq": chunk_type}}
+                else:
+                    chunk_filter = {
+                        "$and": [
+                            filter,
+                            {"chunk_type": {"$eq": chunk_type}}
+                        ]
+                    }
+                
+                try:
+                    chunk_items = self.vectorstore.similarity_search_with_score(
+                        query, filter=chunk_filter, k=search_top
+                    )
+                    
+                    logger.debug(f"Chunk items for {chunk_type}: {chunk_items[0]}")
+                    
+                    for doc, score in chunk_items:
+                        # Create unique identifier for document
+                        source = doc.metadata.get('source')
+                        chunk_id = doc.metadata.get('chunk_id')
+                        doc_id = f"{source}_{chunk_id}" if source and chunk_id else str(doc.metadata.get('id', id(doc)))
+                        
+                        # Store document and its score
+                        if doc_id not in unique_docs:
+                            unique_docs[doc_id] = doc
+                            chunk_type_scores[doc_id] = score
+                            # Create a filter with proper operators 
+                            doc_filter_parts = [
+                                {"source": {"$eq": source}},
+                                {"chunk_id": {"$eq": chunk_id}},
+                                {"chunk_type": {"$eq": "document"}}
+                            ]
+                            
+                            if filter is not None:
+                                doc_filter = {
+                                    "$and": [filter] + doc_filter_parts
+                                }
+                            else:
+                                doc_filter = {
+                                    "$and": doc_filter_parts
+                                }
+                                
+                            try:
+                                fetch_items = self.vectorstore.similarity_search_with_score(
+                                    query, filter=doc_filter, k=1
+                                )
+                                if fetch_items:
+                                    vector_items.append(fetch_items[0])
+
+                            except Exception as e:
+                                logger.warning(f"Error retrieving document chunk for {source}_{chunk_id}: {str(e)}")
+                except Exception as e:
+                    logger.warning(f"Error searching for chunk type {chunk_type}: {str(e)}")
+            
+        else:
+            # Default search behavior (unchanged)
+            max_search_results = 30 if search_top * 3 > 30 else search_top * 3
+            vector_items = self.vectorstore.similarity_search_with_score(
+                query, filter=filter, k=max_search_results
+            )
+            
+        # Initialize document map for tracking by ID
+        doc_map = {
+            f"{doc.metadata.get('id', f'idx_{i}')}_{doc.metadata['chunk_id']}"
+            if 'chunk_id' in doc.metadata
+            else doc.metadata.get('id', f"idx_{i}"): (doc, score)
+            for i, (doc, score) in enumerate(vector_items)
+        }
+        
+        # Process full-text search if configured
+        if full_text_search and full_text_search.get('enabled') and full_text_search.get('fields'):
+            language = full_text_search.get('language', 'english')
+            self._init_pg_helper(language)
+            if self.pg_helper:
+                vector_weight = 1.0  # Default vector weight
+                text_weight = full_text_search.get('weight', 0.3)
+                
+                # Query each specified field
+                for field_name in full_text_search.get('fields', []):
+                    try:
+                        text_results = self.pg_helper.full_text_search(field_name, query)
+                        
+                        # Combine text search results with vector results
+                        for result in text_results:
+                            doc_id = result['id']
+                            text_score = result['text_score']
+                            
+                            if doc_id in doc_map:
+                                # Document exists in vector results, combine scores
+                                doc, vector_score = doc_map[doc_id]
+                                combined_score = (vector_score * vector_weight) + (text_score * text_weight)
+                                doc_map[doc_id] = (doc, combined_score)
+                            else:
+                                # Document is new from text search, fetch and add if possible
+                                doc_data = self.pg_helper.get_documents_by_ids([doc_id]).get(doc_id)
+                                if doc_data:
+                                    from langchain_core.documents import Document
+                                    doc = Document(
+                                        page_content=doc_data.get('document', ''),
+                                        metadata=doc_data.get('cmetadata', {})
+                                    )
+                                    # Use weighted text score for new documents
+                                    doc_map[doc_id] = (doc, text_score * text_weight)
+                    except Exception as e:
+                        logger.error(f"Full-text search error on field {field_name}: {str(e)}")
+            
+        # Convert the document map back to a list
+        combined_items = list(doc_map.values())
+        
+        # Apply reranking rules
+        if reranking_config:
+            combined_items = self._apply_reranking(combined_items, reranking_config)
+        elif reranker:  # Fallback to legacy reranker parameter
+            combined_items = self._apply_reranking(combined_items, reranker)
+        
+        # Apply cutoff filter
+        if cut_off:
+            combined_items = [item for item in combined_items if abs(item[1]) >= cut_off]
+        
+        # Sort by score and limit results
+        # DISABLED: for chroma we want ascending order (lower score is better), for others descending
+        # combined_items.sort(key=lambda x: x[1], reverse= self.vectorstore_type.lower() != 'chroma')
+        combined_items = combined_items[:search_top]
+
+        # Format output based on doctype
+        if doctype == 'code':
+            return code_format(combined_items)
+        else:
+            response = []
+            for doc, score in combined_items:
+                response.append({
+                    'page_content': doc.page_content,
+                    'metadata': doc.metadata,
+                    'score': score
+                })
+            return response
+
+    def _apply_reranking(self, items, reranker):
+        """Apply reranking rules to search results"""
+        if not items:
+            return items
+        
+        # Create a copy of items with mutable scores for reranking
+        reranked_items = [(doc, score) for doc, score in items]
+        
+        for field_name, config in reranker.items():
+            weight = config.get("weight", 1.0)
+            rules = config.get("rules", {})
+            
+            for i, (doc, score) in enumerate(reranked_items):
+                metadata = doc.metadata
+                field_value = metadata.get(field_name)
+                
+                if field_value is not None:
+                    # Apply rules-based reranking
+                    for rule_type, rule_value in rules.items():
+                        if rule_type == "contains" and isinstance(rule_value, str) and isinstance(field_value, str):
+                            if rule_value.lower() in field_value.lower():
+                                # Boost score if field contains the rule value
+                                reranked_items[i] = (doc, score * (1 + weight))
+                        
+                        elif rule_type == "priority":
+                            # Apply priority rule based on exact match
+                            if str(field_value).lower() == str(rule_value).lower():
+                                reranked_items[i] = (doc, score * (1 + weight))
+        
+        # Handle sort rules after individual score adjustments
+        for field_name, config in reranker.items():
+            rules = config.get("rules", {})
+            if "sort" in rules:
+                sort_direction = rules["sort"]
+                # Assuming sort can be "asc" or "desc"
+                reverse_sort = sort_direction.lower() == "desc"
+                
+                # Sort based on the specified field
+                reranked_items.sort(
+                    key=lambda x: (x[0].metadata.get(field_name, None) is not None, 
+                                  x[0].metadata.get(field_name, ""), 
+                                  x[1]),
+                    reverse=reverse_sort
+                )
+        
+        # Re-sort by score if no sort rules were applied
+        if not any("sort" in config.get("rules", {}) for config in reranker.values()):
+            reranked_items.sort(key=lambda x: x[1], reverse=True)
+        
+        return reranked_items
+
+    def stepback_search(self, query:str, messages: list, doctype: str = 'code', 
+                        filter:dict={}, cut_off: float=0.5, search_top:int=10, 
+                        full_text_search: Optional[Dict[str, Any]] = None,
+                        reranking_config: Optional[Dict[str, Dict[str, Any]]] = None,
+                        extended_search: Optional[List[str]] = None):
+        """Enhanced stepback search using JSON configs for full-text search and reranking"""
+        result = self.llm.invoke([
+            HumanMessage(
+                content=[
+                    {
+                        "type": "text", 
+                        "text": STEPBACK_PROMPT.format(input=query, messages=messages)
+                    }
+                ]
+            )
+        ])
+        search_results = self.search_documents(
+            result.content, doctype, filter, cut_off, search_top, 
+            full_text_search=full_text_search,
+            reranking_config=reranking_config,
+            extended_search=extended_search
+        )
+        return search_results
+
+    def stepback_summary(self, query:str, messages: list, doctype: str = 'code', 
+                         filter:dict={}, cut_off: float=0.5, search_top:int=10, 
+                         full_text_search: Optional[Dict[str, Any]] = None,
+                         reranking_config: Optional[Dict[str, Dict[str, Any]]] = None,
+                         extended_search: Optional[List[str]] = None):
+        """Enhanced stepback summary using JSON configs for full-text search and reranking"""
+        search_results = self.stepback_search(
+            query, messages, doctype, filter, cut_off, search_top, 
+            full_text_search=full_text_search,
+            reranking_config=reranking_config,
+            extended_search=extended_search
+        )
+        result = self.llm.invoke([
+            HumanMessage(
+                content=[
+                    {
+                        "type": "text", 
+                        "text": GET_ANSWER_PROMPT.format(input=query, search_results=search_results, messages=messages)
+                    }
+                ]
+            )
+        ])
+        return result.content
+
+    def _log_data(self, message: str, tool_name: str = "index_data"):
+        """Log data and dispatch custom event for indexing progress"""
+
+        try:
+            dispatch_custom_event(
+                name="thinking_step",
+                data={
+                    "message": message,
+                    "tool_name": tool_name,
+                    "toolkit": "vectorstore",
+                },
+            )
+        except Exception as e:
+            logger.warning(f"Failed to dispatch progress event: {str(e)}")
+
+    def get_available_tools(self):
+        return [
+            {
+                "ref": self.index_documents,
+                "name": "indexDocuments",
+                "description": "Index documents in the vectorstore",
+                "args_schema": IndexDocumentsModel
+            },
+            {
+                "ref": self.search_documents,
+                "name": "searchDocuments",
+                "description": "Search documents in the vectorstore",
+                "args_schema": SearchDocumentsModel
+            },
+            {
+                "ref": self.stepback_search,
+                "name": "stepbackSearch",
+                "description": "Search in the vectorstore using stepback technique",
+                "args_schema": StepBackSearchDocumentsModel
+            },
+            {
+                "ref": self.stepback_summary,
+                "name": "stepbackSummary",
+                "description": "Get summary of search results using stepback technique",
+                "args_schema": StepBackSearchDocumentsModel
+            }
+        ]

alita_sdk/tools/ado/work_item/ado_wrapper.py

@@ -3,7 +3,6 @@ import logging
 import urllib.parse
 from typing import Dict, List, Generator, Optional
 
-from alita_sdk.tools.elitea_base import BaseVectorStoreToolApiWrapper, extend_with_vector_tools
 from azure.devops.connection import Connection
 from azure.devops.v7_1.core import CoreClient
 from azure.devops.v7_1.wiki import WikiClient
@@ -15,6 +14,8 @@ from pydantic import create_model, PrivateAttr, SecretStr
 from pydantic import model_validator
 from pydantic.fields import Field
 
+from alita_sdk.tools.non_code_indexer_toolkit import NonCodeIndexerToolkit
+
 try:
     from alita_sdk.runtime.langchain.interfaces.llm_processor import get_embeddings
 except ImportError:
@@ -94,7 +95,7 @@ ADOUnlinkWorkItemsFromWikiPage = create_model(
     page_name=(str, Field(description="Wiki page path to unlink the work items from", examples=["/TargetPage"]))
 )
 
-class AzureDevOpsApiWrapper(BaseVectorStoreToolApiWrapper):
+class AzureDevOpsApiWrapper(NonCodeIndexerToolkit):
     # TODO use ado_configuration instead of organization_url, project and token
     organization_url: str
     project: str
@@ -125,7 +126,7 @@ class AzureDevOpsApiWrapper(BaseVectorStoreToolApiWrapper):
         except Exception as e:
             return ImportError(f"Failed to connect to Azure DevOps: {e}")
 
-        return values
+        return super().validate_toolkit(values)
 
     def _parse_work_items(self, work_items, fields=None):
         """Parse work items dynamically based on the fields requested."""
@@ -522,14 +523,14 @@ class AzureDevOpsApiWrapper(BaseVectorStoreToolApiWrapper):
                 'reason': wi.fields.get('System.Reason', ''),
                 'iteration': wi.fields.get('System.IterationPath', ''),
                 'updated_on': wi.fields.get('System.ChangedDate', ''),
-                'attachment_ids': [rel.url.split('/')[-1] for rel in wi.relations or [] if rel.rel == 'AttachedFile']
+                'attachment_ids': {rel.url.split('/')[-1]:rel.attributes.get('name', '') for rel in wi.relations or [] if rel.rel == 'AttachedFile'}
             })
 
     def _process_document(self, document: Document) -> Generator[Document, None, None]:
-        for attachment_id in document.metadata.get('attachment_ids', []):
+        for attachment_id, file_name in document.metadata.get('attachment_ids', {}).items():
             content_generator = self._client.get_attachment_content(id=attachment_id, download=True)
-            content = ''.join(str(item) for item in content_generator)
-            yield Document(page_content=content, metadata={'id': attachment_id})
+            content = b"".join(x for x in content_generator)
+            yield Document(page_content="", metadata={'id': attachment_id, 'loader_content_type': file_name, 'loader_content': content})
 
     def _index_tool_params(self):
         """Return the parameters for indexing data."""
@@ -537,10 +538,9 @@ class AzureDevOpsApiWrapper(BaseVectorStoreToolApiWrapper):
             "wiql": (str, Field(description="WIQL (Work Item Query Language) query string to select and filter Azure DevOps work items."))
         }
 
-    @extend_with_vector_tools
     def get_available_tools(self):
         """Return a list of available tools."""
-        return [
+        return super().get_available_tools() + [
             {
                 "name": "search_work_items",
                 "description": self.search_work_items.__doc__,

alita_sdk/tools/base_indexer_toolkit.py

@@ -0,0 +1,426 @@
+import json
+import logging
+from typing import Any, Optional, List, Literal, Dict, Generator
+
+from langchain_core.documents import Document
+from pydantic import create_model, Field, SecretStr
+
+# from alita_sdk.runtime.langchain.interfaces.llm_processor import get_embeddings
+from .chunkers import markdown_chunker
+from .utils.content_parser import process_content_by_type
+from .vector_adapters.VectorStoreAdapter import VectorStoreAdapterFactory
+from ..runtime.tools.vectorstore_base import VectorStoreWrapperBase
+from ..runtime.utils.utils import IndexerKeywords
+
+logger = logging.getLogger(__name__)
+
+# Base Vector Store Schema Models
+BaseIndexParams = create_model(
+    "BaseIndexParams",
+    collection_suffix=(str, Field(description="Suffix for collection name (max 7 characters) used to separate datasets", min_length=1, max_length=7)),
+    vectorstore_type=(Optional[str], Field(description="Vectorstore type (Chroma, PGVector, Elastic, etc.)", default="PGVector")),
+)
+
+RemoveIndexParams = create_model(
+    "RemoveIndexParams",
+    collection_suffix=(Optional[str], Field(description="Optional suffix for collection name (max 7 characters)", default="", max_length=7)),
+)
+
+BaseSearchParams = create_model(
+    "BaseSearchParams",
+    query=(str, Field(description="Query text to search in the index")),
+    collection_suffix=(Optional[str], Field(
+        description="Optional suffix for collection name (max 7 characters). Leave empty to search across all datasets",
+        default="", max_length=7)),
+    vectorstore_type=(Optional[str], Field(description="Vectorstore type (Chroma, PGVector, Elastic, etc.)", default="PGVector")),
+    filter=(Optional[dict | str], Field(
+        description="Filter to apply to the search results. Can be a dictionary or a JSON string.",
+        default={},
+        examples=["{\"key\": \"value\"}", "{\"status\": \"active\"}"]
+    )),
+    cut_off=(Optional[float], Field(description="Cut-off score for search results", default=0.5)),
+    search_top=(Optional[int], Field(description="Number of top results to return", default=10)),
+    full_text_search=(Optional[Dict[str, Any]], Field(
+        description="Full text search parameters. Can be a dictionary with search options.",
+        default=None
+    )),
+    extended_search=(Optional[List[str]], Field(
+        description="List of additional fields to include in the search results.",
+        default=None
+    )),
+    reranker=(Optional[dict], Field(
+        description="Reranker configuration. Can be a dictionary with reranking parameters.",
+        default={}
+    )),
+    reranking_config=(Optional[Dict[str, Dict[str, Any]]], Field(
+        description="Reranking configuration. Can be a dictionary with reranking settings.",
+        default=None
+    )),
+)
+
+BaseStepbackSearchParams = create_model(
+    "BaseStepbackSearchParams",
+    query=(str, Field(description="Query text to search in the index")),
+    collection_suffix=(Optional[str], Field(description="Optional suffix for collection name (max 7 characters)", default="", max_length=7)),
+    vectorstore_type=(Optional[str], Field(description="Vectorstore type (Chroma, PGVector, Elastic, etc.)", default="PGVector")),
+    messages=(Optional[List], Field(description="Chat messages for stepback search context", default=[])),
+    filter=(Optional[dict | str], Field(
+        description="Filter to apply to the search results. Can be a dictionary or a JSON string.",
+        default={},
+        examples=["{\"key\": \"value\"}", "{\"status\": \"active\"}"]
+    )),
+    cut_off=(Optional[float], Field(description="Cut-off score for search results", default=0.5)),
+    search_top=(Optional[int], Field(description="Number of top results to return", default=10)),
+    reranker=(Optional[dict], Field(
+        description="Reranker configuration. Can be a dictionary with reranking parameters.",
+        default={}
+    )),
+    full_text_search=(Optional[Dict[str, Any]], Field(
+        description="Full text search parameters. Can be a dictionary with search options.",
+        default=None
+    )),
+    reranking_config=(Optional[Dict[str, Dict[str, Any]]], Field(
+        description="Reranking configuration. Can be a dictionary with reranking settings.",
+        default=None
+    )),
+    extended_search=(Optional[List[str]], Field(
+        description="List of additional fields to include in the search results.",
+        default=None
+    )),
+)
+
+BaseIndexDataParams = create_model(
+    "indexData",
+    __base__=BaseIndexParams,
+    progress_step=(Optional[int], Field(default=10, ge=0, le=100,
+                         description="Optional step size for progress reporting during indexing")),
+    clean_index=(Optional[bool], Field(default=False,
+                       description="Optional flag to enforce clean existing index before indexing new data")),
+    chunking_tool=(Literal[None,'markdown', 'statistical', 'proposal'], Field(description="Name of chunking tool", default=None)),
+    chunking_config=(Optional[dict], Field(description="Chunking tool configuration", default_factory=dict)),
+)
+
+
+class BaseIndexerToolkit(VectorStoreWrapperBase):
+    """Base class for tool API wrappers that support vector store functionality."""
+
+    doctype: str = "document"
+
+    llm: Any = None
+    connection_string: Optional[SecretStr] = None
+    collection_name: Optional[str] = None
+    embedding_model: Optional[str] = "HuggingFaceEmbeddings"
+    embedding_model_params: Optional[Dict[str, Any]] = {"model_name": "sentence-transformers/all-MiniLM-L6-v2"}
+    vectorstore_type: Optional[str] = "PGVector"
+    _embedding: Optional[Any] = None
+    alita: Any = None # Elitea client, if available
+
+    def __init__(self, **kwargs):
+        conn = kwargs.get('connection_string', None)
+        connection_string = conn.get_secret_value() if isinstance(conn, SecretStr) else conn
+        collection_name = kwargs.get('collection_name')
+        
+        # if 'embedding_model' not in kwargs:
+        kwargs['embedding_model'] = 'HuggingFaceEmbeddings'
+        if 'embedding_model_params' not in kwargs:
+            kwargs['embedding_model_params'] = {"model_name": "sentence-transformers/all-MiniLM-L6-v2"}
+        if 'vectorstore_type' not in kwargs:
+            kwargs['vectorstore_type'] = 'PGVector'
+        vectorstore_type = kwargs.get('vectorstore_type')
+        kwargs['vectorstore_params'] = VectorStoreAdapterFactory.create_adapter(vectorstore_type).get_vectorstore_params(collection_name, connection_string)
+        kwargs['_embedding'] = kwargs.get('alita').get_embeddings(kwargs.get('embedding_model'))
+        super().__init__(**kwargs)
+
+    def _index_tool_params(self, **kwargs) -> dict[str, tuple[type, Field]]:
+        """
+        Returns a list of fields for index_data args schema.
+        NOTE: override this method in subclasses to provide specific parameters for certain toolkit.
+        """
+        return {}
+
+    def _base_loader(self, **kwargs) -> Generator[Document, None, None]:
+        """ Loads documents from a source, processes them,
+        and returns a list of Document objects with base metadata: id and created_on."""
+        pass
+
+    def _process_document(self, base_document: Document) -> Generator[Document, None, None]:
+        """ Process an existing base document to extract relevant metadata for full document preparation.
+        Used for late processing of documents after we ensure that the document has to be indexed to avoid
+        time-consuming operations for documents which might be useless.
+
+        Args:
+            document (Document): The base document to process.
+
+        Returns:
+            Document: The processed document with metadata."""
+        pass
+
+    def index_data(self, **kwargs):
+        collection_suffix = kwargs.get("collection_suffix")
+        progress_step = kwargs.get("progress_step")
+        clean_index = kwargs.get("clean_index")
+        chunking_tool = kwargs.get("chunking_tool")
+        chunking_config = kwargs.get("chunking_config")
+        #
+        if clean_index:
+            self._clean_index()
+        #
+        documents = self._base_loader(**kwargs)
+        documents = self._reduce_duplicates(documents, collection_suffix)
+        documents = self._extend_data(documents) # update content of not-reduced base document if needed (for sharepoint and similar)
+        documents = self._collect_dependencies(documents) # collect dependencies for base documents
+        documents = self._apply_loaders_chunkers(documents, chunking_tool, chunking_config)
+        #
+        return self._save_index(list(documents), collection_suffix=collection_suffix, progress_step=progress_step)
+    
+    def _apply_loaders_chunkers(self, documents: Generator[Document, None, None], chunking_tool: str=None, chunking_config=None) -> Generator[Document, None, None]:
+        from alita_sdk.tools.chunkers import __confluence_chunkers__ as chunkers, __confluence_models__ as models
+
+        if chunking_config is None:
+            chunking_config = {}
+        chunking_config['embedding'] = self._embedding
+        chunking_config['llm'] = self.llm
+            
+        for document in documents:
+            if content_type := document.metadata.get('loader_content_type', None):
+                # apply parsing based on content type and chunk if chunker was applied to parent doc
+                yield from process_content_by_type(
+                    document=document,
+                    extension_source=content_type, llm=self.llm, chunking_config=chunking_config)
+            elif chunking_tool:
+                # apply default chunker from toolkit config. No parsing.
+                chunker = chunkers.get(chunking_tool)
+                yield from chunker(file_content_generator=iter([document]), config=chunking_config)
+            else:
+                # return as is if neither chunker or content typa are specified
+                yield document
+    
+    def _extend_data(self, documents: Generator[Document, None, None]):
+        yield from documents
+
+    def _collect_dependencies(self, documents: Generator[Document, None, None]):
+        for document in documents:
+            dependencies = self._process_document(document)
+            yield document
+            for dep in dependencies:
+                dep.metadata[IndexerKeywords.PARENT.value] = document.metadata.get('id', None)
+                yield dep
+    
+    def _content_loader(self):
+        pass
+
+    def _reduce_duplicates(
+            self,
+            documents: Generator[Any, None, None],
+            collection_suffix: str,
+            log_msg: str = "Verification of documents to index started"
+    ) -> Generator[Document, None, None]:
+        """Generic duplicate reduction logic for documents."""
+        self._log_data(log_msg, tool_name="index_documents")
+        indexed_data = self._get_indexed_data(collection_suffix)
+        indexed_keys = set(indexed_data.keys())
+        if not indexed_keys:
+            self._log_data("Vectorstore is empty, indexing all incoming documents", tool_name="index_documents")
+            yield from documents
+            return
+
+        docs_to_remove = set()
+
+        for document in documents:
+            key = self.key_fn(document)
+            if key in indexed_keys and collection_suffix == indexed_data[key]['metadata'].get('collection'):
+                if self.compare_fn(document, indexed_data[key]):
+                    continue
+                yield document
+                docs_to_remove.update(self.remove_ids_fn(indexed_data, key))
+            else:
+                yield document
+
+        if docs_to_remove:
+            self._log_data(
+                f"Removing {len(docs_to_remove)} documents from vectorstore that are already indexed with different updated_on.",
+                tool_name="index_documents"
+            )
+            self.vectorstore.delete(ids=list(docs_to_remove))
+    
+    def _get_indexed_data(self, collection_suffix: str):
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def key_fn(self, document: Document):
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def compare_fn(self, document: Document, idx):
+        raise NotImplementedError("Subclasses must implement this method")
+    
+    def remove_ids_fn(self, idx_data, key: str):
+        raise NotImplementedError("Subclasses must implement this method")
+
+    def _process_documents(self, documents: List[Document]) -> Generator[Document, None, None]:
+        """
+        Process a list of base documents to extract relevant metadata for full document preparation.
+        Used for late processing of documents after we ensure that the documents have to be indexed to avoid
+        time-consuming operations for documents which might be useless.
+        This function passed to index_documents method of vector store and called after _reduce_duplicates method.
+
+        Args:
+            documents (List[Document]): The base documents to process.
+
+        Returns:
+            Generator[Document, None, None]: A generator yielding processed documents with metadata.
+        """
+        for doc in documents:
+            # Filter documents to process only those that either:
+            # - do not have a 'chunk_id' in their metadata, or
+            # - have 'chunk_id' explicitly set to 1.
+            # This prevents processing of irrelevant or duplicate chunks, improving efficiency.
+            chunk_id = doc.metadata.get("chunk_id")
+            if chunk_id is None or chunk_id == 1:
+                processed_docs = self._process_document(doc)
+                if processed_docs:  # Only proceed if the list is not empty
+                    for processed_doc in processed_docs:
+                        # map processed document (child) to the original document (parent)
+                        processed_doc.metadata[IndexerKeywords.PARENT.value] = doc.metadata.get('id', None)
+                        if chunker:=self._get_dependencies_chunker(processed_doc):
+                            yield from chunker(file_content_generator=iter([processed_doc]), config=self._get_dependencies_chunker_config())
+                        else:
+                            yield processed_doc
+
+    def remove_index(self, collection_suffix: str = ""):
+        """Cleans the indexed data in the collection."""
+        super()._clean_collection(collection_suffix=collection_suffix)
+        return (f"Collection '{collection_suffix}' has been removed from the vector store.\n"
+                f"Available collections: {self.list_collections()}")
+
+    def search_index(self,
+                     query: str,
+                     collection_suffix: str = "",
+                     filter: dict | str = {}, cut_off: float = 0.5,
+                     search_top: int = 10, reranker: dict = {},
+                     full_text_search: Optional[Dict[str, Any]] = None,
+                     reranking_config: Optional[Dict[str, Dict[str, Any]]] = None,
+                     extended_search: Optional[List[str]] = None,
+                     **kwargs):
+        """ Searches indexed documents in the vector store."""
+        # build filter on top of collection_suffix
+        filter = filter if isinstance(filter, dict) else json.loads(filter)
+        if collection_suffix:
+            filter.update({"collection": {
+                "$eq": collection_suffix.strip()
+            }})
+
+        found_docs = super().search_documents(
+            query,
+            doctype=self.doctype,
+            filter=filter,
+            cut_off=cut_off,
+            search_top=search_top,
+            reranker=reranker,
+            full_text_search=full_text_search,
+            reranking_config=reranking_config,
+            extended_search=extended_search
+        )
+        return found_docs if found_docs else f"No documents found by query '{query}' and filter '{filter}'"
+
+    def stepback_search_index(self,
+                     query: str,
+                     messages: List[Dict[str, Any]] = [],
+                     collection_suffix: str = "",
+                     filter: dict | str = {}, cut_off: float = 0.5,
+                     search_top: int = 10, reranker: dict = {},
+                     full_text_search: Optional[Dict[str, Any]] = None,
+                     reranking_config: Optional[Dict[str, Dict[str, Any]]] = None,
+                     extended_search: Optional[List[str]] = None,
+                     **kwargs):
+        """ Searches indexed documents in the vector store."""
+        found_docs = super().stepback_search(
+            query,
+            messages,
+            self.doctype,
+            filter=filter,
+            cut_off=cut_off,
+            search_top=search_top,
+            full_text_search=full_text_search,
+            reranking_config=reranking_config,
+            extended_search=extended_search
+        )
+        return f"Found {len(found_docs)} documents matching the query\n{json.dumps(found_docs, indent=4)}" if found_docs else "No documents found matching the query."
+
+    def stepback_summary_index(self,
+                     query: str,
+                     messages: List[Dict[str, Any]] = [],
+                     collection_suffix: str = "",
+                     filter: dict | str = {}, cut_off: float = 0.5,
+                     search_top: int = 10, reranker: dict = {},
+                     full_text_search: Optional[Dict[str, Any]] = None,
+                     reranking_config: Optional[Dict[str, Dict[str, Any]]] = None,
+                     extended_search: Optional[List[str]] = None,
+                     **kwargs):
+        """ Generates a summary of indexed documents using stepback technique."""
+        return super().stepback_summary(
+            query, 
+            messages, 
+            self.doctype, 
+            filter=filter, 
+            cut_off=cut_off, 
+            search_top=search_top,
+            full_text_search=full_text_search, 
+            reranking_config=reranking_config, 
+            extended_search=extended_search
+        )
+
+    def get_available_tools(self):
+        """
+        Returns the standardized vector search tools (search operations only).
+        Index operations are toolkit-specific and should be added manually to each toolkit.
+        
+        Returns:
+            List of tool dictionaries with name, ref, description, and args_schema
+        """
+        return [
+            {
+                "name": "index_data",
+                "mode": "index_data",
+                "ref": self.index_data,
+                "description": "Loads data to index.",
+                "args_schema": create_model(
+                    "IndexData",
+                    __base__=BaseIndexDataParams,
+                    **self._index_tool_params() if self._index_tool_params() else {}
+                )
+            },
+            {
+                "name": "search_index",
+                "mode": "search_index",
+                "ref": self.search_index,
+                "description": self.search_index.__doc__,
+                "args_schema": BaseSearchParams
+            },
+            {
+                "name": "stepback_search_index",
+                "mode": "stepback_search_index",
+                "ref": self.stepback_search_index,
+                "description": self.stepback_search_index.__doc__,
+                "args_schema": BaseStepbackSearchParams
+            },
+            {
+                "name": "stepback_summary_index",
+                "mode": "stepback_summary_index",
+                "ref": self.stepback_summary_index,
+                "description": self.stepback_summary_index.__doc__,
+                "args_schema": BaseStepbackSearchParams
+            },
+            {
+                "name": "remove_index",
+                "mode": "remove_index",
+                "ref": self.remove_index,
+                "description": self.remove_index.__doc__,
+                "args_schema": RemoveIndexParams
+            },
+            {
+                "name": "list_collections",
+                "mode": "list_collections",
+                "ref": self.list_collections,
+                "description": self.list_collections.__doc__,
+                "args_schema": create_model("ListCollectionsParams")  # No parameters
+            },
+        ]

alita_sdk/tools/elitea_base.py

@@ -31,13 +31,11 @@ LoaderSchema = create_model(
 BaseIndexParams = create_model(
     "BaseIndexParams",
     collection_suffix=(str, Field(description="Suffix for collection name (max 7 characters) used to separate datasets", min_length=1, max_length=7)),
-    vectorstore_type=(Optional[str], Field(description="Vectorstore type (Chroma, PGVector, Elastic, etc.)", default="PGVector")),
 )
 
 BaseCodeIndexParams = create_model(
     "BaseCodeIndexParams",
     collection_suffix=(str, Field(description="Suffix for collection name (max 7 characters) used to separate datasets", min_length=1, max_length=7)),
-    vectorstore_type=(Optional[str], Field(description="Vectorstore type (Chroma, PGVector, Elastic, etc.)", default="PGVector")),
     branch=(Optional[str], Field(description="Branch to index files from. Defaults to active branch if None.", default=None)),
     whitelist=(Optional[List[str]], Field(description="File extensions or paths to include. Defaults to all files if None.", default=None)),
     blacklist=(Optional[List[str]], Field(description="File extensions or paths to exclude. Defaults to no exclusions if None.", default=None)),
@@ -54,7 +52,6 @@ BaseSearchParams = create_model(
     collection_suffix=(Optional[str], Field(
         description="Optional suffix for collection name (max 7 characters). Leave empty to search across all datasets",
         default="", max_length=7)),
-    vectorstore_type=(Optional[str], Field(description="Vectorstore type (Chroma, PGVector, Elastic, etc.)", default="PGVector")),
     filter=(Optional[dict | str], Field(
         description="Filter to apply to the search results. Can be a dictionary or a JSON string.",
         default={},
@@ -84,7 +81,6 @@ BaseStepbackSearchParams = create_model(
     "BaseStepbackSearchParams",
     query=(str, Field(description="Query text to search in the index")),
     collection_suffix=(Optional[str], Field(description="Optional suffix for collection name (max 7 characters)", default="", max_length=7)),
-    vectorstore_type=(Optional[str], Field(description="Vectorstore type (Chroma, PGVector, Elastic, etc.)", default="PGVector")),
     messages=(Optional[List], Field(description="Chat messages for stepback search context", default=[])),
     filter=(Optional[dict | str], Field(
         description="Filter to apply to the search results. Can be a dictionary or a JSON string.",
@@ -572,12 +568,14 @@ class BaseCodeToolApiWrapper(BaseVectorStoreToolApiWrapper):
 
         def is_whitelisted(file_path: str) -> bool:
             if whitelist:
-                return any(fnmatch.fnmatch(file_path, pattern) for pattern in whitelist)
+                return (any(fnmatch.fnmatch(file_path, pattern) for pattern in whitelist)
+                        or any(file_path.endswith(f'.{pattern}') for pattern in whitelist))
             return True
 
         def is_blacklisted(file_path: str) -> bool:
             if blacklist:
-                return any(fnmatch.fnmatch(file_path, pattern) for pattern in blacklist)
+                return (any(fnmatch.fnmatch(file_path, pattern) for pattern in blacklist)
+                        or any(file_path.endswith(f'.{pattern}') for pattern in blacklist))
             return False
 
         def file_content_generator():

alita_sdk/tools/non_code_indexer_toolkit.py

@@ -0,0 +1,23 @@
+from langchain_core.documents import Document
+
+from alita_sdk.runtime.utils.utils import IndexerKeywords
+from alita_sdk.tools.base_indexer_toolkit import BaseIndexerToolkit
+
+
+class NonCodeIndexerToolkit(BaseIndexerToolkit):
+    def _get_indexed_data(self, collection_suffix: str):
+        return self.vector_adapter.get_indexed_data(self, collection_suffix)
+
+    def key_fn(self, document: Document):
+        return document.metadata.get('id')
+
+    def compare_fn(self, document: Document, idx_data):
+        return (document.metadata.get('updated_on')
+                and idx_data['metadata'].get('updated_on')
+                and document.metadata.get('updated_on') == idx_data['metadata'].get('updated_on'))
+
+    def remove_ids_fn(self, idx_data, key: str):
+        return (idx_data[key]['all_chunks'] +
+                [idx_data[dep_id]['id'] for dep_id in idx_data[key][IndexerKeywords.DEPENDENT_DOCS.value]] +
+                [chunk_db_id for dep_id in idx_data[key][IndexerKeywords.DEPENDENT_DOCS.value] for chunk_db_id in
+                 idx_data[dep_id]['all_chunks']])

alita_sdk/tools/utils/content_parser.py

@@ -1,11 +1,16 @@
 import os
 import tempfile
+from copy import deepcopy as copy
 from logging import getLogger
 from pathlib import Path
+from typing import Generator
 
+from langchain_core.documents import Document
 from langchain_core.tools import ToolException
+from langchain_text_splitters import TokenTextSplitter
 
 from alita_sdk.runtime.langchain.document_loaders.constants import loaders_map
+from alita_sdk.tools.chunkers.utils import tiktoken_length
 
 logger = getLogger(__name__)
 
@@ -161,6 +166,61 @@ def load_content_from_bytes(file_content: bytes, extension: str = None, loader_e
         # Now the file is closed and can be read
         result = load_content(temp_file_path, extension, loader_extra_config, llm)
         return result
+    finally:
+        if temp_file_path and os.path.exists(temp_file_path):
+            os.remove(temp_file_path)
+
+def process_content_by_type(document: Document, extension_source: str, llm = None, chunking_config={}) -> Generator[Document, None, None]:
+    temp_file_path = None
+    try:
+        extension = "." + extension_source.split('.')[-1].lower()
+
+        with tempfile.NamedTemporaryFile(mode='w+b', suffix=extension, delete=False) as temp_file:
+            temp_file_path = temp_file.name
+            content = document.metadata.pop('loader_content')
+            temp_file.write(content)
+            temp_file.flush()
+
+            loader_config = loaders_map.get(extension)
+            if not loader_config:
+                logger.warning(f"No loader found for file extension: {extension}. File: {temp_file_path}")
+                return
+
+            loader_cls = loader_config['class']
+            loader_kwargs = loader_config['kwargs']
+
+            loader = loader_cls(file_path=temp_file_path, **loader_kwargs)
+            docs_iterator = loader.load()
+            max_tokens = chunking_config.get('max_tokens', 512)
+            tokens_overlapping = chunking_config.get('tokens_overlapping', 10)
+            chunk_id = 0
+            for chunk in docs_iterator:
+                if tiktoken_length(chunk.page_content) > max_tokens:
+                    for subchunk in TokenTextSplitter(encoding_name="cl100k_base", 
+                                                      chunk_size=max_tokens, 
+                                                      chunk_overlap=tokens_overlapping
+                                                      ).split_text(chunk.page_content):
+                        chunk_id += 1
+                        headers_meta = list(chunk.metadata.values())
+                        docmeta = copy(document.metadata)
+                        docmeta.update({"headers": "; ".join(str(headers_meta))})
+                        docmeta['chunk_id'] = chunk_id
+                        docmeta['chunk_type'] = "document"
+                        yield Document(
+                            page_content=subchunk,
+                            metadata=docmeta
+                        )
+                else:
+                    chunk_id += 1
+                    headers_meta = list(chunk.metadata.values())
+                    docmeta = copy(document.metadata)
+                    docmeta.update({"headers": "; ".join(str(headers_meta))})
+                    docmeta['chunk_id'] = chunk_id
+                    docmeta['chunk_type'] = "document"
+                    yield Document(
+                        page_content=chunk.page_content,
+                        metadata=docmeta
+                    )
     finally:
         if temp_file_path and os.path.exists(temp_file_path):
             os.remove(temp_file_path)

{alita_sdk-0.3.249.dist-info → alita_sdk-0.3.250.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.249
+Version: 0.3.250
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedjik@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.249.dist-info → alita_sdk-0.3.250.dist-info}/RECORD

@@ -106,6 +106,7 @@ alita_sdk/runtime/tools/prompt.py,sha256=nJafb_e5aOM1Rr3qGFCR-SKziU9uCsiP2okIMs9
 alita_sdk/runtime/tools/router.py,sha256=wCvZjVkdXK9dMMeEerrgKf5M790RudH68pDortnHSz0,1517
 alita_sdk/runtime/tools/tool.py,sha256=lE1hGi6qOAXG7qxtqxarD_XMQqTghdywf261DZawwno,5631
 alita_sdk/runtime/tools/vectorstore.py,sha256=l5wfovwMNvS_RgW-ZHXCh8Cm8gauunRzP0NPkzmshcQ,33852
+alita_sdk/runtime/tools/vectorstore_base.py,sha256=OdJIJkjTmQ0BC-AzAOMP2phAcNATJ8gI5JoBWSSdpNU,27892
 alita_sdk/runtime/utils/AlitaCallback.py,sha256=E4LlSBuCHWiUq6W7IZExERHZY0qcmdjzc_rJlF2iQIw,7356
 alita_sdk/runtime/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 alita_sdk/runtime/utils/constants.py,sha256=Xntx1b_uxUzT4clwqHA_U6K8y5bBqf_4lSQwXdcWrp4,13586
@@ -117,7 +118,9 @@ alita_sdk/runtime/utils/toolkit_runtime.py,sha256=MU63Fpxj0b5_r1IUUc0Q3-PN9VwL7r
 alita_sdk/runtime/utils/toolkit_utils.py,sha256=I9QFqnaqfVgN26LUr6s3XlBlG6y0CoHURnCzG7XcwVs,5311
 alita_sdk/runtime/utils/utils.py,sha256=CpEl3LCeLbhzQySz08lkKPm7Auac6IiLF7WB8wmArMI,589
 alita_sdk/tools/__init__.py,sha256=ko5TToGYZFmBrho26DRAVvrkHWxQ2sfs8gVAASinYp8,10611
-alita_sdk/tools/elitea_base.py,sha256=A88HXZUMdoE_YtYVY3JwhgpKWaA_LIFTu54Ms2pj_Fs,30688
+alita_sdk/tools/base_indexer_toolkit.py,sha256=qQfMHzsQ2BfusKMV_DNiHOtZVheiQ4gBfy5JXjYi0UY,20231
+alita_sdk/tools/elitea_base.py,sha256=kmfVA965-IkcDBnGQzfv3E14sD6RXLDzJFcj7CJ3fDc,30350
+alita_sdk/tools/non_code_indexer_toolkit.py,sha256=v9uq1POE1fQKCd152mbqDtF-HSe0qoDj83k4E5LAkMI,1080
 alita_sdk/tools/ado/__init__.py,sha256=bArTObt5cqG1SkijKevWGbsIILHBA3aCStg8Q1jd69k,1243
 alita_sdk/tools/ado/utils.py,sha256=PTCludvaQmPLakF2EbCGy66Mro4-rjDtavVP-xcB2Wc,1252
 alita_sdk/tools/ado/repos/__init__.py,sha256=_vjU3yHRXmLg6BDNmJsLiM9qDYRE_JmX5kXI_irMmQQ,5789
@@ -127,7 +130,7 @@ alita_sdk/tools/ado/test_plan/test_plan_wrapper.py,sha256=jQt8kFmdAzsopjByLTMiSn
 alita_sdk/tools/ado/wiki/__init__.py,sha256=uBKo_Meu2ZxMxcxGsMmvCXyplRE2um1_PIRvdYd37rM,5171
 alita_sdk/tools/ado/wiki/ado_wrapper.py,sha256=zg6wMRar1DTp-ZRlYaQifBEnpYmTrHXskTNPdrLdy8s,14759
 alita_sdk/tools/ado/work_item/__init__.py,sha256=coDedNL0pSPLjZ6VVK1UcqWo00zxe2T4XfVXt8bMho8,5383
-alita_sdk/tools/ado/work_item/ado_wrapper.py,sha256=ubeF2m8J6CGZF_gnkTEbmW_eh6YWsk7bD2clu9FmZpY,28313
+alita_sdk/tools/ado/work_item/ado_wrapper.py,sha256=gEywCL_kS0k1jWcDhsmYUybpIP08tH8go6CixLJGwT4,28409
 alita_sdk/tools/advanced_jira_mining/__init__.py,sha256=pUTzECqGvYaR5qWY3JPUhrImrZgc7pCXuqSe5eWIE80,4604
 alita_sdk/tools/advanced_jira_mining/data_mining_wrapper.py,sha256=nZPtuwVWp8VeHw1B8q9kdwf-6ZvHnlXTOGdcIMDkKpw,44211
 alita_sdk/tools/aws/__init__.py,sha256=tB6GCOg4XGSpR6qgbgAF4MUQ5-YmQCbWurWgrVKEKQ8,181
@@ -308,7 +311,7 @@ alita_sdk/tools/testio/api_wrapper.py,sha256=BvmL5h634BzG6p7ajnQLmj-uoAw1gjWnd4F
 alita_sdk/tools/testrail/__init__.py,sha256=0kETjWKLU7R6mugBWsjwEUsh10pipbAeNSGJAO0FBh0,4634
 alita_sdk/tools/testrail/api_wrapper.py,sha256=K-Gc42RH2z-fK4cXi8zQq3s9A4v_pCJkRB3XKLAhypc,32056
 alita_sdk/tools/utils/__init__.py,sha256=155xepXPr4OEzs2Mz5YnjXcBpxSv1X2eznRUVoPtyK0,3268
-alita_sdk/tools/utils/content_parser.py,sha256=uqJoaJzl9w4Nf9yFy40sm1-qm88bvj7Y1S-1OxHkTks,7410
+alita_sdk/tools/utils/content_parser.py,sha256=0HKQqGTdXHKlcz72GHEwXqLXJsRYXm35F-P1KZz0sNc,10351
 alita_sdk/tools/vector_adapters/VectorStoreAdapter.py,sha256=a6FAsiix_EvATIKUf5YT6vHh5LDyJ5uSP3LJqoxFo04,17367
 alita_sdk/tools/vector_adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 alita_sdk/tools/xray/__init__.py,sha256=GGpbiBdDQ9kMFqJEHYi7XwKpkuMMHi-ZF-IM8yFIgUM,4380
@@ -330,8 +333,8 @@ alita_sdk/tools/zephyr_scale/api_wrapper.py,sha256=JAeWf-RXohsxheUpT0iMDClc_izj-
 alita_sdk/tools/zephyr_squad/__init__.py,sha256=0AI_j27xVO5Gk5HQMFrqPTd4uvuVTpiZUicBrdfEpKg,2796
 alita_sdk/tools/zephyr_squad/api_wrapper.py,sha256=kmw_xol8YIYFplBLWTqP_VKPRhL_1ItDD0_vXTe_UuI,14906
 alita_sdk/tools/zephyr_squad/zephyr_squad_cloud_client.py,sha256=R371waHsms4sllHCbijKYs90C-9Yu0sSR3N4SUfQOgU,5066
-alita_sdk-0.3.249.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-alita_sdk-0.3.249.dist-info/METADATA,sha256=dDaElnaGjY8jmdfIQ3k8wXejBDlZUVR7A1U_kA6rwEs,18897
-alita_sdk-0.3.249.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-alita_sdk-0.3.249.dist-info/top_level.txt,sha256=0vJYy5p_jK6AwVb1aqXr7Kgqgk3WDtQ6t5C-XI9zkmg,10
-alita_sdk-0.3.249.dist-info/RECORD,,
+alita_sdk-0.3.250.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+alita_sdk-0.3.250.dist-info/METADATA,sha256=vKpZbYSYxt7ruy33ugylQXj66_5YWm3Cuw3Wz-psGSA,18897
+alita_sdk-0.3.250.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+alita_sdk-0.3.250.dist-info/top_level.txt,sha256=0vJYy5p_jK6AwVb1aqXr7Kgqgk3WDtQ6t5C-XI9zkmg,10
+alita_sdk-0.3.250.dist-info/RECORD,,