alita-sdk 0.3.257__py3-none-any.whl → 0.3.584__py3-none-any.whl

alita_sdk/tools/browser/utils.py

@@ -6,9 +6,9 @@ from langchain.text_splitter import CharacterTextSplitter
 import fitz
 
 try:
-    from langchain_chroma import Chroma
+    from langchain_postgres import PGVector
 except ImportError:
-    Chroma = None
+    PGVector = None
 
 from langchain_community.embeddings.sentence_transformer import (
     SentenceTransformerEmbeddings,
@@ -32,13 +32,22 @@ def get_page(urls, html_only=False):
     return docs_transformed
 
 
-def webRag(urls, max_response_size, query):
-    if Chroma is None:
-        return "Chroma is not initialized. Web rag is not available."
+def webRag(urls, max_response_size, query, connection_string=None):
+    if PGVector is None:
+        return "PGVector is not initialized. Web rag is not available."
+
+    if not connection_string:
+        return "Connection string or embedding model is missing. Web rag is not available."
     text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
     docs = text_splitter.split_documents(get_page(urls))
     embedding_function = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
-    db = Chroma.from_documents(docs, embedding_function)
+    db = PGVector.from_documents(
+        documents=docs,
+        embedding=embedding_function,
+        collection_name="web_rag",
+        pre_delete_collection=True,
+        connection=connection_string
+    )
     docs = db.search(query, "mmr", k=10)
     text = ""
     for doc in docs:

alita_sdk/tools/carrier/__init__.py

@@ -1,12 +1,14 @@
 import logging
 from typing import Dict, List, Optional, Literal
 from langchain_core.tools import BaseToolkit, BaseTool
-from pydantic import create_model, BaseModel, ConfigDict, Field, SecretStr
+from pydantic import create_model, BaseModel, ConfigDict, Field
 from functools import lru_cache
 
 from .api_wrapper import CarrierAPIWrapper
 from .tools import __all__
-from ..utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
+from ..elitea_base import filter_missconfigured_index_tools
+from ..utils import clean_string, get_max_toolkit_length
+from ...configurations.carrier import CarrierConfiguration
 
 logger = logging.getLogger(__name__)
 
@@ -15,7 +17,6 @@ name = 'carrier'
 
 class AlitaCarrierToolkit(BaseToolkit):
     tools: List[BaseTool] = []
-    toolkit_max_length: int = 100
 
     @classmethod
     @lru_cache(maxsize=32)
@@ -24,15 +25,10 @@ class AlitaCarrierToolkit(BaseToolkit):
         for t in __all__:
             default = t['tool'].__pydantic_fields__['args_schema'].default
             selected_tools[t['name']] = default.schema() if default else default
-        cls.toolkit_max_length = get_max_toolkit_length(selected_tools)
         return create_model(
             name,
-            url=(str, Field(description="Carrier Platform Base URL")),
-            organization=(str, Field(description="Carrier Organization Name", json_schema_extra={'toolkit_name': True,
-                                                                                                 'max_toolkit_length': cls.toolkit_max_length})),
-            private_token=(
-                SecretStr, Field(description="Carrier Platform Authentication Token", json_schema_extra={'secret': True})),
             project_id=(Optional[str], Field(None, description="Optional project ID for scoped operations")),
+            carrier_configuration=(CarrierConfiguration, Field(description="Carrier Configuration", json_schema_extra={'configuration_types': ['carrier']})),
             selected_tools=(
                 List[Literal[tuple(selected_tools)]],
                 Field(default=[], json_schema_extra={"args_schemas": selected_tools}),
@@ -49,6 +45,7 @@ class AlitaCarrierToolkit(BaseToolkit):
         )
 
     @classmethod
+    @filter_missconfigured_index_tools
     def get_toolkit(
             cls,
             selected_tools: Optional[List[str]] = None,
@@ -58,23 +55,29 @@ class AlitaCarrierToolkit(BaseToolkit):
         selected_tools = selected_tools or []
         logger.info(f"[AlitaCarrierToolkit] Initializing toolkit with selected tools: {selected_tools}")
 
+        wrapper_payload = {
+            **kwargs,
+            **kwargs.get('carrier_configuration', {}),
+        }
+
         try:
-            carrier_api_wrapper = CarrierAPIWrapper(**kwargs)
+            carrier_api_wrapper = CarrierAPIWrapper(**wrapper_payload)
             logger.info(
-                f"[AlitaCarrierToolkit] CarrierAPIWrapper initialized successfully with URL: {kwargs.get('url')}")
+                f"[AlitaCarrierToolkit] CarrierAPIWrapper initialized successfully with URL: {wrapper_payload.get('url')}")
         except Exception as e:
             logger.exception(f"[AlitaCarrierToolkit] Error initializing CarrierAPIWrapper: {e}")
             raise ValueError(f"CarrierAPIWrapper initialization error: {e}")
 
-        prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
-
         tools = []
         for tool_def in __all__:
             if selected_tools and tool_def['name'] not in selected_tools:
                 continue
             try:
                 tool_instance = tool_def['tool'](api_wrapper=carrier_api_wrapper)
-                tool_instance.name = prefix + tool_instance.name
+                if toolkit_name:
+                    tool_instance.description = f"{tool_instance.description}\nToolkit: {toolkit_name}"
+                    tool_instance.description = tool_instance.description[:1000]
+                    tool_instance.metadata = {"toolkit_name": toolkit_name, "toolkit_type": name}
                 tools.append(tool_instance)
                 logger.info(f"[AlitaCarrierToolkit] Successfully initialized tool '{tool_instance.name}'")
             except Exception as e:
@@ -92,9 +95,7 @@ class AlitaCarrierToolkit(BaseToolkit):
 def get_tools(tool_config: Dict) -> List[BaseTool]:
     return AlitaCarrierToolkit.get_toolkit(
         selected_tools=tool_config.get('selected_tools', []),
-        url=tool_config['settings']['url'],
         project_id=tool_config['settings'].get('project_id'),
-        organization=tool_config['settings']['organization'],
-        private_token=tool_config['settings']['private_token'],
+        carrier_configuration=tool_config['settings']['carrier_configuration'],
         toolkit_name=tool_config.get('toolkit_name')
     ).get_tools()

alita_sdk/tools/carrier/backend_reports_tool.py

@@ -154,6 +154,7 @@ class CreateExcelReportTool(BaseTool):
             "tp_threshold": (int, Field(default=None, description="Throughput threshold")),
             "rt_threshold": (int, Field(default=None, description="Response time threshold")),
             "er_threshold": (int, Field(default=None, description="Error rate threshold")),
+            "include_group_pauses": (bool, Field(default=False, description="Include group pauses in Gatling Excel report")),
         }
     )
 
@@ -200,6 +201,7 @@ class CreateExcelReportTool(BaseTool):
             "tp_threshold": 10,
             "rt_threshold": 500,
             "er_threshold": 5,
+            "include_group_pauses": False,
         }
 
     def _request_parameter_confirmation(self, default_parameters):
@@ -217,7 +219,8 @@ class CreateExcelReportTool(BaseTool):
         excel_report_file_name = f'/tmp/reports_test_results_{report["build_id"]}_excel_report.xlsx'
         bucket_name = report["name"].replace("_", "").replace(" ", "").lower()
 
-        result_stats_j = self._parse_report(test_log_file_path, lg_type, parameters["think_time"], is_absolute_file_path=True)
+        result_stats_j = self._parse_report(test_log_file_path, lg_type, parameters["think_time"],
+                                            parameters["include_group_pauses"], is_absolute_file_path=True)
         calc_thr_j = self._calculate_thresholds(result_stats_j, parameters)
 
         return self._generate_and_upload_report(
@@ -233,21 +236,22 @@ class CreateExcelReportTool(BaseTool):
         excel_report_file_name = f'{file_path}_{current_date}.xlsx'
         bucket_name = bucket
 
-        result_stats_j = self._parse_report(file_path, lg_type, parameters["think_time"], is_absolute_file_path=True)
+        result_stats_j = self._parse_report(file_path, lg_type, parameters["think_time"],
+                                            parameters["include_group_pauses"], is_absolute_file_path=True)
         calc_thr_j = self._calculate_thresholds(result_stats_j, parameters)
 
         return self._generate_and_upload_report(
             result_stats_j, carrier_report, calc_thr_j, parameters, excel_report_file_name, bucket_name, file_path
         )
 
-    def _parse_report(self, file_path, lg_type, think_time, is_absolute_file_path=False):
+    def _parse_report(self, file_path, lg_type, think_time, include_group_pauses, is_absolute_file_path=False):
         """Parse the report based on its type."""
         if lg_type == "gatling":
             if is_absolute_file_path:
                 report_file = file_path
             else:
                 report_file = get_latest_log_file(file_path, "simulation.log")
-            parser = GatlingReportParser(report_file, think_time)
+            parser = GatlingReportParser(report_file, include_group_pauses, think_time)
             result_stats_j = parser.parse()
             result_stats_j["requests"].update(result_stats_j["groups"])
         elif lg_type == "jmeter":

alita_sdk/tools/carrier/excel_reporter.py

@@ -118,9 +118,10 @@ class JMeterReportParser(PerformanceReportParser):
 
 class GatlingReportParser(PerformanceReportParser):
 
-    def __init__(self, log_file: str, think_times="5,0-10,0"):
+    def __init__(self, log_file: str, include_group_pauses, think_times="5,0-10,0"):
         self.calculated_think_time = think_times
         self.log_file = log_file
+        self.include_group_pauses = include_group_pauses
 
     @staticmethod
     def convert_timestamp_to_datetime(timestamp: int) -> datetime:
@@ -210,7 +211,7 @@ class GatlingReportParser(PerformanceReportParser):
                                 ramp_end = self.convert_timestamp_to_datetime(int(line.split('\t')[3]))
 
                     elif line.startswith('GROUP'):
-                        self.parse_group_line(groups, line)
+                        self.parse_group_line(groups, line, self.include_group_pauses)
         except FileNotFoundError as e:
             print(f"File not found: {e}")
             raise
@@ -242,11 +243,14 @@ class GatlingReportParser(PerformanceReportParser):
             requests[request_name].append((response_time, status))
 
     @staticmethod
-    def parse_group_line(groups, line):
+    def parse_group_line(groups, line, include_group_pauses):
         parts = line.split('\t')
         if len(parts) >= 6:
             group_name = parts[1]
-            response_time = int(parts[4])
+            if include_group_pauses:
+                response_time = int(parts[3]) - int(parts[2])
+            else:
+                response_time = int(parts[4])
             status = parts[5].strip()
             groups[group_name].append((response_time, status))
 

alita_sdk/tools/chunkers/__init__.py

@@ -3,6 +3,7 @@ from .sematic.statistical_chunker import statistical_chunker
 from .sematic.markdown_chunker import markdown_chunker
 from .sematic.proposal_chunker import proposal_chunker
 from .sematic.json_chunker import json_chunker
+from .universal_chunker import universal_chunker, chunk_single_document, get_file_type
 from .models import StatisticalChunkerConfig, MarkdownChunkerConfig, ProposalChunkerConfig
 
 __all__ = {
@@ -10,7 +11,8 @@ __all__ = {
     'statistical': statistical_chunker,
     'markdown': markdown_chunker,
     'proposal': proposal_chunker,
-    'json': json_chunker
+    'json': json_chunker,
+    'universal': universal_chunker,
 }
 
 __confluence_chunkers__ = {

alita_sdk/tools/chunkers/code/codeparser.py

@@ -79,7 +79,7 @@ def parse_code_files_for_db(file_content_generator: Generator[str, None, None],
                     for splitted_document in splitted_documents:
                         metadata = {
                             "filename": file_name,
-                            "method_name": node.name,
+                            "method_name": node.name if node.name else 'unknown',
                             "language": programming_language.value,
                         }
                         commit_hash = data.get("commit_hash")

alita_sdk/tools/chunkers/sematic/json_chunker.py

@@ -9,7 +9,7 @@ def json_chunker(file_content_generator: Generator[Document, None, None], config
     for doc in file_content_generator:
         try:
             data_dict = json.loads(doc.page_content)
-            chunks = RecursiveJsonSplitter(max_chunk_size=max_tokens).split_json(json_data=data_dict)
+            chunks = RecursiveJsonSplitter(max_chunk_size=max_tokens).split_json(json_data=data_dict, convert_lists=True)
             if len(chunks) == 1:
                 yield doc
                 continue
@@ -17,6 +17,7 @@ def json_chunker(file_content_generator: Generator[Document, None, None], config
             for chunk in chunks:
                 metadata = doc.metadata.copy()
                 metadata['chunk_id'] = chunk_id
+                metadata['method_name'] = 'json'
                 chunk_id += 1
                 yield Document(page_content=json.dumps(chunk), metadata=metadata)
         except Exception as e:

alita_sdk/tools/chunkers/sematic/markdown_chunker.py

@@ -1,4 +1,4 @@
-from typing import Generator
+from typing import Generator, List
 from langchain_core.documents import Document
 from langchain_text_splitters import MarkdownHeaderTextSplitter, ExperimentalMarkdownSyntaxTextSplitter
 from langchain.text_splitter import TokenTextSplitter
@@ -7,34 +7,60 @@ from copy import deepcopy as copy
 
 
 def markdown_chunker(file_content_generator: Generator[Document, None, None], config: dict, *args, **kwargs) -> Generator[Document, None, None]:
+    """
+    Chunks markdown documents by headers, with support for:
+    - Minimum chunk size to avoid tiny fragments
+    - Maximum token limit with overflow splitting
+    - Header metadata preservation
+    
+    Config options:
+        strip_header (bool): Remove headers from content. Default: False
+        return_each_line (bool): Split on every line. Default: False
+        headers_to_split_on (list): Headers to split on, e.g. [('#', 'H1'), ('##', 'H2')]
+        max_tokens (int): Maximum tokens per chunk. Default: 512
+        token_overlap (int): Token overlap for large chunk splitting. Default: 10
+        min_chunk_chars (int): Minimum characters per chunk. Default: 100
+            Chunks smaller than this will be merged with the next chunk.
+    """
     strip_header = config.get("strip_header", False)
     return_each_line = config.get("return_each_line", False)
     headers_to_split_on = config.get("headers_to_split_on", [])
     max_tokens = config.get("max_tokens", 512)
     tokens_overlapping = config.get("token_overlap", 10)
+    min_chunk_chars = config.get("min_chunk_chars", 100)  # Minimum characters per chunk
+    
     headers_to_split_on = [tuple(header) for header in headers_to_split_on]
+    
     for doc in file_content_generator:
         doc_metadata = doc.metadata
         doc_content = doc.page_content
         chunk_id = 0
+        
         markdown_splitter = MarkdownHeaderTextSplitter(
             headers_to_split_on=headers_to_split_on, 
             strip_headers=strip_header,
             return_each_line=return_each_line
         )
         md_header_splits = markdown_splitter.split_text(doc_content)
-        for chunk in md_header_splits:
+        
+        # Merge small chunks with the next one
+        merged_chunks = _merge_small_chunks(md_header_splits, min_chunk_chars)
+        
+        for chunk in merged_chunks:
             if tiktoken_length(chunk.page_content) > max_tokens:
-                for subchunk in TokenTextSplitter(encoding_name="cl100k_base", 
-                                                  chunk_size=max_tokens, 
-                                                  chunk_overlap=tokens_overlapping
-                                                  ).split_text(chunk.page_content):
+                # Split large chunks into smaller ones
+                for subchunk in TokenTextSplitter(
+                    encoding_name="cl100k_base", 
+                    chunk_size=max_tokens, 
+                    chunk_overlap=tokens_overlapping
+                ).split_text(chunk.page_content):
                     chunk_id += 1
                     headers_meta = list(chunk.metadata.values())
                     docmeta = copy(doc_metadata)
                     docmeta.update({"headers": "; ".join(headers_meta)})
                     docmeta['chunk_id'] = chunk_id
                     docmeta['chunk_type'] = "document"
+                    docmeta['method_name'] = 'markdown'
                     yield Document(
                         page_content=subchunk,
                         metadata=docmeta
@@ -46,12 +72,77 @@ def markdown_chunker(file_content_generator: Generator[Document, None, None], co
                 docmeta.update({"headers": "; ".join(headers_meta)})
                 docmeta['chunk_id'] = chunk_id
                 docmeta['chunk_type'] = "document"
+                docmeta['method_name'] = 'text'
                 yield Document(
                     page_content=chunk.page_content,
                     metadata=docmeta
                 )
 
 
+def _merge_small_chunks(chunks: List[Document], min_chars: int) -> List[Document]:
+    """
+    Merge chunks that are smaller than min_chars with the next chunk.
+    
+    This prevents tiny fragments (like standalone headers or short notes)
+    from becoming separate chunks.
+    
+    Args:
+        chunks: List of Document chunks from markdown splitter
+        min_chars: Minimum character count for a chunk
+        
+    Returns:
+        List of merged Document chunks
+    """
+    if not chunks:
+        return chunks
+    
+    merged = []
+    pending_content = ""
+    pending_metadata = {}
+    
+    for i, chunk in enumerate(chunks):
+        content = chunk.page_content.strip()
+        
+        if pending_content:
+            # Merge pending content with current chunk
+            combined_content = pending_content + "\n\n" + content
+            # Use the pending metadata (from the header) but can be extended
+            combined_metadata = {**pending_metadata}
+            # Add any new header info from current chunk
+            for key, value in chunk.metadata.items():
+                if key not in combined_metadata or not combined_metadata[key]:
+                    combined_metadata[key] = value
+            
+            if len(combined_content) >= min_chars:
+                # Combined is big enough, emit it
+                merged.append(Document(
+                    page_content=combined_content,
+                    metadata=combined_metadata
+                ))
+                pending_content = ""
+                pending_metadata = {}
+            else:
+                # Still too small, keep accumulating
+                pending_content = combined_content
+                pending_metadata = combined_metadata
+        elif len(content) < min_chars:
+            # Current chunk is too small, start pending
+            pending_content = content
+            pending_metadata = dict(chunk.metadata)
+        else:
+            # Current chunk is big enough
+            merged.append(chunk)
+    
+    # Don't forget any remaining pending content
+    if pending_content:
+        merged.append(Document(
+            page_content=pending_content,
+            metadata=pending_metadata
+        ))
+    
+    return merged
+
+
 def markdown_by_headers_chunker(file_content_generator: Generator[Document, None, None], config: dict, *args, **kwargs) -> Generator[Document, None, None]:
     strip_header = config.get("strip_header", False)
     return_each_line = config.get("return_each_line", False)

alita_sdk/tools/chunkers/sematic/proposal_chunker.py

@@ -6,7 +6,7 @@ from langchain_core.prompts import ChatPromptTemplate
 from langchain.text_splitter import TokenTextSplitter
 
 from typing import Optional, List
-from langchain_core.pydantic_v1 import BaseModel
+from pydantic import BaseModel
 from ..utils import tiktoken_length
 
 logger = getLogger(__name__)