alita-sdk 0.3.329__py3-none-any.whl → 0.3.330__py3-none-any.whl

alita_sdk/runtime/langchain/document_loaders/AlitaTextLoader.py

@@ -1,4 +1,4 @@
-from typing import Iterator
+from typing import Iterator, Generator
 
 from langchain_core.documents import Document
 
@@ -6,6 +6,9 @@ from langchain_community.document_loaders.base import BaseLoader
 from langchain_community.document_loaders.helpers import detect_file_encodings
 from langchain_core.tools import ToolException
 
+from alita_sdk.tools.chunkers import markdown_chunker
+
+
 class AlitaTextLoader(BaseLoader):
 
     def __init__(self, **kwargs):
@@ -19,6 +22,8 @@ class AlitaTextLoader(BaseLoader):
             raise ToolException("'file_path' or 'file_content' parameter should be provided.")
         self.encoding = kwargs.get('encoding', 'utf-8')
         self.autodetect_encoding = kwargs.get('autodetect_encoding', False)
+        self.max_tokens=kwargs.get('max_tokens', 1024)
+        self.token_overlap = kwargs.get('token_overlap', 10)
 
     def get_content(self):
         text = ""
@@ -59,8 +64,16 @@ class AlitaTextLoader(BaseLoader):
 
         return text
 
+    def generate_document(self, text, metadata) -> Generator[Document, None, None]:
+        yield Document(page_content=text, metadata=metadata)
+
     def lazy_load(self) -> Iterator[Document]:
         """Load from file path."""
         text = self.get_content()
         metadata = {"source": str(self.file_path) if hasattr(self, 'file_path') else self.file_name}
-        yield Document(page_content=text, metadata=metadata)
+        chunks = markdown_chunker(file_content_generator=self.generate_document(text, metadata),
+                                  config={
+                                      "max_tokens": self.max_tokens,
+                                      "token_overlap": self.token_overlap
+                                  })
+        yield from chunks

alita_sdk/tools/utils/content_parser.py

@@ -1,4 +1,5 @@
 import os
+import re
 import tempfile
 from logging import getLogger
 from pathlib import Path
@@ -8,6 +9,7 @@ from langchain_core.documents import Document
 from langchain_core.tools import ToolException
 
 from alita_sdk.runtime.langchain.document_loaders.constants import loaders_map, LoaderProperties
+from ...runtime.langchain.document_loaders.AlitaTextLoader import AlitaTextLoader
 from ...runtime.utils.utils import IndexerKeywords
 
 logger = getLogger(__name__)
@@ -231,7 +233,8 @@ def process_content_by_type(content, filename: str, llm=None, chunking_config=No
     """Process the content of a file based on its type using a configured loader."""
     temp_file_path = None
     try:
-        extension = "." + filename.split('.')[-1].lower()
+        match = re.search(r'\.([^.]+)$', filename)
+        extension = f".{match.group(1).lower()}" if match else ".txt"
 
         with tempfile.NamedTemporaryFile(mode='w+b', suffix=extension, delete=False) as temp_file:
             temp_file_path = temp_file.name

{alita_sdk-0.3.329.dist-info → alita_sdk-0.3.330.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.329
+Version: 0.3.330
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedj27@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.329.dist-info → alita_sdk-0.3.330.dist-info}/RECORD

@@ -67,7 +67,7 @@ alita_sdk/runtime/langchain/document_loaders/AlitaPowerPointLoader.py,sha256=SKA
 alita_sdk/runtime/langchain/document_loaders/AlitaPythonLoader.py,sha256=m_7aq-aCFVb4vXZsJNinfN1hAuyy_S0ylRknv_ahxDc,340
 alita_sdk/runtime/langchain/document_loaders/AlitaQtestLoader.py,sha256=CUVVnisxm7b5yZWV6rn0Q3MEEaO1GWNcfnz5yWz8T0k,13283
 alita_sdk/runtime/langchain/document_loaders/AlitaTableLoader.py,sha256=nI8lyndVZxVAxbjX3yiqyuFQKFE8MjLPyYSyqRWxHqQ,4077
-alita_sdk/runtime/langchain/document_loaders/AlitaTextLoader.py,sha256=uNcV0En49_0u0RYB1sP1XfNspT2Xc5CacuJr9Jqv79Q,2972
+alita_sdk/runtime/langchain/document_loaders/AlitaTextLoader.py,sha256=EiCIAF_OxSrbuwgOFk2IpxRMvFbctITt2jAI0g_atpk,3586
 alita_sdk/runtime/langchain/document_loaders/ImageParser.py,sha256=gao5yCCKdDai_Gx7YdEx5U6oMyJYzn69eYmEvWLh-fc,656
 alita_sdk/runtime/langchain/document_loaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 alita_sdk/runtime/langchain/document_loaders/constants.py,sha256=H5oKHDHZw1L0x9sGiYGLmkGM6dH460bvuj-EycmpR6E,7235
@@ -327,7 +327,7 @@ alita_sdk/tools/testrail/__init__.py,sha256=Xg4nVjULL_D8JpIXLYXppnwUfGF4-lguFwKH
 alita_sdk/tools/testrail/api_wrapper.py,sha256=PKhtf04C6PFDexGCAJm-hjA9Gpu4crx6EXKT5K-b_Pk,32985
 alita_sdk/tools/utils/__init__.py,sha256=W9rCCUPtHCP5nGAbWp0n5jaNA84572aiRoqKneBnaS4,3330
 alita_sdk/tools/utils/available_tools_decorator.py,sha256=IbrdfeQkswxUFgvvN7-dyLMZMyXLiwvX7kgi3phciCk,273
-alita_sdk/tools/utils/content_parser.py,sha256=SLwRNNb2oahxM5DW9MJ570NgEFDKXl53bbMZ9021Ee0,14238
+alita_sdk/tools/utils/content_parser.py,sha256=ZgjzopMY3OyEcypa8NEwB4263HZ08V0fKQZRJXPmlj4,14393
 alita_sdk/tools/vector_adapters/VectorStoreAdapter.py,sha256=ypBEAkFRGHv5edW0N9rdo1yKurNGQ4pRVEWtrN_7SeA,17656
 alita_sdk/tools/vector_adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 alita_sdk/tools/xray/__init__.py,sha256=eOMWP8VamFbbJgt1xrGpGPqB9ByOTA0Cd3LCaETzGk4,4376
@@ -349,8 +349,8 @@ alita_sdk/tools/zephyr_scale/api_wrapper.py,sha256=kT0TbmMvuKhDUZc0i7KO18O38JM9S
 alita_sdk/tools/zephyr_squad/__init__.py,sha256=0ne8XLJEQSLOWfzd2HdnqOYmQlUliKHbBED5kW_Vias,2895
 alita_sdk/tools/zephyr_squad/api_wrapper.py,sha256=kmw_xol8YIYFplBLWTqP_VKPRhL_1ItDD0_vXTe_UuI,14906
 alita_sdk/tools/zephyr_squad/zephyr_squad_cloud_client.py,sha256=R371waHsms4sllHCbijKYs90C-9Yu0sSR3N4SUfQOgU,5066
-alita_sdk-0.3.329.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-alita_sdk-0.3.329.dist-info/METADATA,sha256=BRvAazGNKqm2sPsodjbdSv7A6pfFCEzX_l1jtj7c4DM,18835
-alita_sdk-0.3.329.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-alita_sdk-0.3.329.dist-info/top_level.txt,sha256=0vJYy5p_jK6AwVb1aqXr7Kgqgk3WDtQ6t5C-XI9zkmg,10
-alita_sdk-0.3.329.dist-info/RECORD,,
+alita_sdk-0.3.330.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+alita_sdk-0.3.330.dist-info/METADATA,sha256=tTJb4beJGhlKTftR_US7mWBMNQ4PaJbyxkp2bJ2AxOg,18835
+alita_sdk-0.3.330.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+alita_sdk-0.3.330.dist-info/top_level.txt,sha256=0vJYy5p_jK6AwVb1aqXr7Kgqgk3WDtQ6t5C-XI9zkmg,10
+alita_sdk-0.3.330.dist-info/RECORD,,