alita-sdk 0.3.264__tar.gz → 0.3.266__tar.gz

{alita_sdk-0.3.264 → alita_sdk-0.3.266}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.264
+Version: 0.3.266
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedjik@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.264 → alita_sdk-0.3.266}/alita_sdk/runtime/tools/artifact.py

@@ -9,6 +9,7 @@ from pydantic import create_model, Field, model_validator
 
 from alita_sdk.tools.non_code_indexer_toolkit import NonCodeIndexerToolkit
 from alita_sdk.tools.utils.available_tools_decorator import extend_with_parent_available_tools
+from ...runtime.utils.utils import IndexerKeywords
 
 
 class ArtifactWrapper(NonCodeIndexerToolkit):
@@ -82,8 +83,8 @@ class ArtifactWrapper(NonCodeIndexerToolkit):
         for document in documents:
             try:
                 page_content = self.artifact.get_content_bytes(artifact_name=document.metadata['name'])
-                document.metadata['loader_content'] = page_content
-                document.metadata['loader_content_type'] = document.metadata['name']
+                document.metadata[IndexerKeywords.CONTENT_IN_BYTES.value] = page_content
+                document.metadata[IndexerKeywords.CONTENT_FILE_NAME.value] = document.metadata['name']
                 yield document
             except Exception as e:
                 logging.error(f"Failed while parsing the file '{document.metadata['name']}': {e}")

{alita_sdk-0.3.264 → alita_sdk-0.3.266}/alita_sdk/runtime/utils/utils.py

@@ -9,6 +9,8 @@ class IndexerKeywords(Enum):
     PARENT = 'parent_id'
     # DEPENDENCY_ID = 'dependency_id'
     UPDATED_ON = 'updated_on'
+    CONTENT_IN_BYTES = 'loader_content'
+    CONTENT_FILE_NAME = 'loader_content_type'
 
 # This pattern matches characters that are NOT alphanumeric, underscores, or hyphens
 clean_string_pattern = re.compile(r'[^a-zA-Z0-9_.-]')

{alita_sdk-0.3.264 → alita_sdk-0.3.266}/alita_sdk/tools/ado/test_plan/test_plan_wrapper.py

@@ -1,7 +1,7 @@
 import json
 import logging
 import xml.etree.ElementTree as ET
-from typing import Generator, Optional
+from typing import Generator, Literal, Optional
 
 from azure.devops.connection import Connection
 from azure.devops.v7_0.test_plan.models import TestPlanCreateParams, TestSuiteCreateParams, \
@@ -14,7 +14,9 @@ from pydantic import create_model, PrivateAttr, model_validator, SecretStr
 from pydantic.fields import FieldInfo as Field
 
 from ..work_item import AzureDevOpsApiWrapper
-from ...elitea_base import BaseVectorStoreToolApiWrapper, extend_with_vector_tools
+from ...non_code_indexer_toolkit import NonCodeIndexerToolkit
+from ...utils.available_tools_decorator import extend_with_parent_available_tools
+from ....runtime.utils.utils import IndexerKeywords
 
 logger = logging.getLogger(__name__)
 
@@ -158,7 +160,7 @@ TestCasesGetModel = create_model(
     suite_id=(int, Field(description="ID of the test suite for which test cases are requested"))
 )
 
-class TestPlanApiWrapper(BaseVectorStoreToolApiWrapper):
+class TestPlanApiWrapper(NonCodeIndexerToolkit):
     # TODO use ado_configuration instead of organization_url, project and token
     __test__ = False
     organization_url: str
@@ -178,7 +180,7 @@ class TestPlanApiWrapper(BaseVectorStoreToolApiWrapper):
             cls._client = connection.clients.get_test_plan_client()
         except Exception as e:
             raise ImportError(f"Failed to connect to Azure DevOps: {e}")
-        return values
+        return super().validate_toolkit(values)
 
     def create_test_plan(self, test_plan_create_params: str):
         """Create a test plan in Azure DevOps."""
@@ -360,7 +362,7 @@ class TestPlanApiWrapper(BaseVectorStoreToolApiWrapper):
             logger.error(f"Error getting test cases: {e}")
             return ToolException(f"Error getting test cases: {e}")
 
-    def _base_loader(self, plan_id: str, suite_ids: Optional[list[str]] = [], **kwargs) -> Generator[Document, None, None]:
+    def _base_loader(self, plan_id: str, suite_ids: Optional[list[str]] = [], chunking_tool: str = None, **kwargs) -> Generator[Document, None, None]:
         cases = []
         for sid in suite_ids:
             cases.extend(self.get_test_cases(plan_id, sid))
@@ -368,29 +370,39 @@ class TestPlanApiWrapper(BaseVectorStoreToolApiWrapper):
         for case in cases:
             field_dicts = case.get('work_item', {}).get('work_item_fields', [])
             data = {k: v for d in field_dicts for k, v in d.items()}
-            yield Document(
-                page_content=data.get('Microsoft.VSTS.TCM.Steps', ''),
-                metadata={
-                    'id': case.get('work_item', {}).get('id', ''),
-                    'title': case.get('work_item', {}).get('name', ''),
-                    'plan_id': case.get('test_plan', {}).get('id', ''),
-                    'suite_id': case.get('test_suite', {}).get('id', ''),
-                    'description': data.get('System.Description', ''),
-                    'updated_on': data.get('System.Rev', ''),
-                })
-
-    def _process_document(self, document: Document) -> Generator[Document, None, None]:
-        if False:
-            yield  # Unreachable, but keeps the function a generator
+            if chunking_tool:
+                yield Document(
+                    page_content='',
+                    metadata={
+                        'id': case.get('work_item', {}).get('id', ''),
+                        'title': case.get('work_item', {}).get('name', ''),
+                        'plan_id': case.get('test_plan', {}).get('id', ''),
+                        'suite_id': case.get('test_suite', {}).get('id', ''),
+                        'description': data.get('System.Description', ''),
+                        'updated_on': data.get('System.Rev', ''),
+                        IndexerKeywords.CONTENT_IN_BYTES.value: data.get('Microsoft.VSTS.TCM.Steps', '').encode("utf-8")
+                    })
+            else:
+                yield Document(
+                    page_content=data.get('Microsoft.VSTS.TCM.Steps', ''),
+                    metadata={
+                        'id': case.get('work_item', {}).get('id', ''),
+                        'title': case.get('work_item', {}).get('name', ''),
+                        'plan_id': case.get('test_plan', {}).get('id', ''),
+                        'suite_id': case.get('test_suite', {}).get('id', ''),
+                        'description': data.get('System.Description', ''),
+                        'updated_on': data.get('System.Rev', ''),
+                    })
 
     def _index_tool_params(self):
         """Return the parameters for indexing data."""
         return {
             "plan_id": (str, Field(description="ID of the test plan for which test cases are requested")),
-            "suite_ids": (str, Field(description="List of test suite IDs for which test cases are requested (can be empty)"))
+            "suite_ids": (str, Field(description="List of test suite IDs for which test cases are requested (can be empty)")),
+            'chunking_tool':(Literal['html'], Field(description="Name of chunking tool", default='html'))
         }
 
-    @extend_with_vector_tools
+    @extend_with_parent_available_tools
     def get_available_tools(self):
         """Return a list of available tools."""
         return [

{alita_sdk-0.3.264 → alita_sdk-0.3.266}/alita_sdk/tools/ado/wiki/ado_wrapper.py

@@ -1,6 +1,6 @@
 import hashlib
 import logging
-from typing import Any, Dict, Generator, Optional
+from typing import Generator, Literal, Optional
 
 from azure.devops.connection import Connection
 from azure.devops.exceptions import AzureDevOpsServiceError
@@ -15,7 +15,9 @@ from pydantic import create_model, PrivateAttr, SecretStr
 from pydantic import model_validator
 from pydantic.fields import Field
 
-from ...elitea_base import BaseVectorStoreToolApiWrapper, extend_with_vector_tools
+from ...non_code_indexer_toolkit import NonCodeIndexerToolkit
+from ...utils.available_tools_decorator import extend_with_parent_available_tools
+from ....runtime.utils.utils import IndexerKeywords
 
 logger = logging.getLogger(__name__)
 
@@ -55,7 +57,7 @@ RenamePageInput = create_model(
 )
 
 
-class AzureDevOpsApiWrapper(BaseVectorStoreToolApiWrapper):
+class AzureDevOpsApiWrapper(NonCodeIndexerToolkit):
     # TODO use ado_configuration instead of organization_url, project and token
     organization_url: str
     project: str
@@ -82,7 +84,7 @@ class AzureDevOpsApiWrapper(BaseVectorStoreToolApiWrapper):
         except Exception as e:
             return ImportError(f"Failed to connect to Azure DevOps: {e}")
 
-        return values
+        return super().validate_toolkit(values)
 
     def get_wiki(self, wiki_identified: str):
         """Extract ADO wiki information."""
@@ -219,7 +221,7 @@ class AzureDevOpsApiWrapper(BaseVectorStoreToolApiWrapper):
             logger.error(f"Unable to modify wiki page: {str(e)}")
             return ToolException(f"Unable to modify wiki page: {str(e)}")
 
-    def _base_loader(self, wiki_identifier: str, title_contains: Optional[str] = None, **kwargs) -> Generator[Document, None, None]:
+    def _base_loader(self, wiki_identifier: str, chunking_tool: str = None, title_contains: Optional[str] = None, **kwargs) -> Generator[Document, None, None]:
         pages = self._client.get_pages_batch(pages_batch_request={}, project=self.project, wiki_identifier=wiki_identifier)
         #
         for page in pages:
@@ -227,21 +229,31 @@ class AzureDevOpsApiWrapper(BaseVectorStoreToolApiWrapper):
             content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()
             title = page.path.rsplit("/", 1)[-1]
             if not title_contains or (title_contains and title_contains.lower() in title.lower()):
-                yield Document(page_content=content, metadata={
-                    'id': str(page.id),
-                    'path': page.path,
-                    'title': title,
-                    'updated_on': content_hash
-                })
+                if chunking_tool:
+                    yield Document(page_content='', metadata={
+                        'id': str(page.id),
+                        'path': page.path,
+                        'title': title,
+                        'updated_on': content_hash,
+                        IndexerKeywords.CONTENT_IN_BYTES.value: content.encode("utf-8")
+                    })
+                else:
+                    yield Document(page_content=content, metadata={
+                        'id': str(page.id),
+                        'path': page.path,
+                        'title': title,
+                        'updated_on': content_hash
+                    })
 
     def _index_tool_params(self):
         """Return the parameters for indexing data."""
         return {
             "wiki_identifier": (str, Field(description="Wiki identifier to index, e.g., 'ABCProject.wiki'")),
-            'title_contains': (Optional[str], Field(default=None, description="Optional filter to include only pages with titles containing exact this string"))
+            'title_contains': (Optional[str], Field(default=None, description="Optional filter to include only pages with titles containing exact this string")),
+            'chunking_tool':(Literal['markdown'], Field(description="Name of chunking tool", default='markdown'))
         }
 
-    @extend_with_vector_tools
+    @extend_with_parent_available_tools
     def get_available_tools(self):
         """Return a list of available tools."""
         return [

{alita_sdk-0.3.264 → alita_sdk-0.3.266}/alita_sdk/tools/ado/work_item/ado_wrapper.py

@@ -15,6 +15,7 @@ from pydantic import model_validator
 from pydantic.fields import Field
 
 from alita_sdk.tools.non_code_indexer_toolkit import NonCodeIndexerToolkit
+from ....runtime.utils.utils import IndexerKeywords
 
 logger = logging.getLogger(__name__)
 
@@ -525,7 +526,7 @@ class AzureDevOpsApiWrapper(NonCodeIndexerToolkit):
         for attachment_id, file_name in document.metadata.get('attachment_ids', {}).items():
             content_generator = self._client.get_attachment_content(id=attachment_id, download=True)
             content = b"".join(x for x in content_generator)
-            yield Document(page_content="", metadata={'id': attachment_id, 'loader_content_type': file_name, 'loader_content': content})
+            yield Document(page_content="", metadata={'id': attachment_id, IndexerKeywords.CONTENT_FILE_NAME.value: file_name, IndexerKeywords.CONTENT_IN_BYTES.value: content})
 
     def _index_tool_params(self):
         """Return the parameters for indexing data."""

{alita_sdk-0.3.264 → alita_sdk-0.3.266}/alita_sdk/tools/base_indexer_toolkit.py

@@ -5,7 +5,7 @@ from typing import Any, Optional, List, Literal, Dict, Generator
 from langchain_core.documents import Document
 from pydantic import create_model, Field, SecretStr
 
-from .utils.content_parser import process_content_by_type
+from .utils.content_parser import file_extension_by_chunker, process_content_by_type
 from .vector_adapters.VectorStoreAdapter import VectorStoreAdapterFactory
 from ..runtime.tools.vectorstore_base import VectorStoreWrapperBase
 from ..runtime.utils.utils import IndexerKeywords
@@ -112,7 +112,10 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
         
         if 'embedding_model' not in kwargs:
             kwargs['embedding_model'] = 'HuggingFaceEmbeddings'
-        kwargs['vectorstore_params'] = VectorStoreAdapterFactory.create_adapter(self.vectorstore_type).get_vectorstore_params(collection_name, connection_string)
+        if 'vectorstore_type' not in kwargs:
+            kwargs['vectorstore_type'] = 'PGVector'
+        vectorstore_type = kwargs.get('vectorstore_type')
+        kwargs['vectorstore_params'] = VectorStoreAdapterFactory.create_adapter(vectorstore_type).get_vectorstore_params(collection_name, connection_string)
         kwargs['_embedding'] = kwargs.get('alita').get_embeddings(kwargs.get('embedding_model'))
         super().__init__(**kwargs)
 
@@ -167,19 +170,26 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
         chunking_config['llm'] = self.llm
         
         for document in documents:
-            if content_type := document.metadata.get('loader_content_type', None):
+            if content_type := document.metadata.get(IndexerKeywords.CONTENT_FILE_NAME.value, None):
                 # apply parsing based on content type and chunk if chunker was applied to parent doc
-                content = document.metadata.pop('loader_content', None)
+                content = document.metadata.pop(IndexerKeywords.CONTENT_IN_BYTES.value, None)
                 yield from process_content_by_type(
                     document=document,
                     content=content,
                     extension_source=content_type, llm=self.llm, chunking_config=chunking_config)
+            elif chunking_tool and (content_in_bytes := document.metadata.pop(IndexerKeywords.CONTENT_IN_BYTES.value, None)):
+                # apply parsing based on content type resolved from chunking_tool
+                content_type = file_extension_by_chunker(chunking_tool)
+                yield from process_content_by_type(
+                    document=document,
+                    content=content_in_bytes,
+                    extension_source=content_type, llm=self.llm, chunking_config=chunking_config)
             elif chunking_tool:
                 # apply default chunker from toolkit config. No parsing.
                 chunker = chunkers.get(chunking_tool)
                 yield from chunker(file_content_generator=iter([document]), config=chunking_config)
             else:
-                # return as is if neither chunker or content typa are specified
+                # return as is if neither chunker nor content type are specified
                 yield document
     
     def _extend_data(self, documents: Generator[Document, None, None]):

{alita_sdk-0.3.264 → alita_sdk-0.3.266}/alita_sdk/tools/sharepoint/api_wrapper.py

@@ -10,6 +10,7 @@ from pydantic import Field, PrivateAttr, create_model, model_validator, SecretSt
 
 from ..non_code_indexer_toolkit import NonCodeIndexerToolkit
 from ..utils.content_parser import parse_file_content
+from ...runtime.utils.utils import IndexerKeywords
 
 NoInput = create_model(
     "NoInput"
@@ -170,7 +171,7 @@ class SharepointApiWrapper(NonCodeIndexerToolkit):
 
     def _base_loader(self, **kwargs) -> Generator[Document, None, None]:
         try:
-            all_files = self.get_files_list(kwargs.get('limit_files', 10000))
+            all_files = self.get_files_list(limit_files=kwargs.get('limit_files', 10000))
         except Exception as e:
             raise ToolException(f"Unable to extract files: {e}")
 
@@ -187,7 +188,7 @@ class SharepointApiWrapper(NonCodeIndexerToolkit):
 
             # Check if file should be included based on include_extensions
             # If include_extensions is empty, process all files (that weren't skipped)
-            if not (any(re.match(pattern.replace('*', '.*') + '$', file_name, re.IGNORECASE)
+            if include_extensions and not (any(re.match(pattern.replace('*', '.*') + '$', file_name, re.IGNORECASE)
                         for pattern in include_extensions)):
                 continue
 
@@ -200,8 +201,8 @@ class SharepointApiWrapper(NonCodeIndexerToolkit):
     def _extend_data(self, documents: Generator[Document, None, None]):
         for document in documents:
             try:
-                document.metadata['loader_content'] = self._load_file_content_in_bytes(document.metadata['Path'])
-                document.metadata['loader_content_type'] = document.metadata['Name']
+                document.metadata[IndexerKeywords.CONTENT_IN_BYTES.value] = self._load_file_content_in_bytes(document.metadata['Path'])
+                document.metadata[IndexerKeywords.CONTENT_FILE_NAME.value] = document.metadata['Name']
                 yield document
             except Exception as e:
                 logging.error(f"Failed while parsing the file '{document.metadata['Path']}': {e}")

{alita_sdk-0.3.264 → alita_sdk-0.3.266}/alita_sdk/tools/utils/content_parser.py

@@ -8,6 +8,7 @@ from langchain_core.documents import Document
 from langchain_core.tools import ToolException
 
 from alita_sdk.runtime.langchain.document_loaders.constants import loaders_map
+from ...runtime.utils.utils import IndexerKeywords
 
 logger = getLogger(__name__)
 
@@ -175,7 +176,7 @@ def process_content_by_type(document: Document, content, extension_source: str,
         with tempfile.NamedTemporaryFile(mode='w+b', suffix=extension, delete=False) as temp_file:
             temp_file_path = temp_file.name
             if content is None:
-                logger.warning("'loader_content' ie expected but not found in document metadata.")
+                logger.warning(f"'{IndexerKeywords.CONTENT_IN_BYTES.value}' ie expected but not found in document metadata.")
                 return
             
             temp_file.write(content)
@@ -218,4 +219,19 @@ def sanitize_for_postgres(text: str, replacement: str = "") -> str:
         >>> sanitize_for_postgres("Hello\\x00world", " ")
         'Hello world'
     """
-    return text.replace("\x00", replacement)
+    return text.replace("\x00", replacement)
+
+
+def file_extension_by_chunker(chunker_name: str) -> str:
+    name = chunker_name.lower()
+    if name == "markdown":
+        return ".md"
+    if name == "json":
+        return ".json"
+    if name == "text" or name == "txt":
+        return ".txt"
+    if name == "html":
+        return ".html"
+    if name == "csv":
+        return ".csv"
+    return None

{alita_sdk-0.3.264 → alita_sdk-0.3.266}/alita_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.264
+Version: 0.3.266
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedjik@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.264 → alita_sdk-0.3.266}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "alita_sdk"
-version = "0.3.264"
+version = "0.3.266"
 description = "SDK for building langchain agents using resources from Alita"
 readme = "README.md"
 requires-python = ">=3.10"