alita-sdk 0.3.374__py3-none-any.whl → 0.3.423__py3-none-any.whl

alita_sdk/tools/sharepoint/authorization_helper.py

@@ -1,7 +1,10 @@
 from datetime import datetime, timezone
+from urllib.parse import unquote, urlparse, quote
 
 import jwt
 import requests
+from botocore.response import get_response
+
 
 class SharepointAuthorizationHelper:
 
@@ -54,4 +57,175 @@ class SharepointAuthorizationHelper:
         except jwt.ExpiredSignatureError:
             return False
         except jwt.InvalidTokenError:
-            return False
+            return False
+
+    def _validate_response(self, response, required_field, error_prefix=None):
+        if response.status_code != 200:
+            raise RuntimeError(f"{error_prefix or 'Request'} failed: {response.status_code} {response.text}")
+        json_data = response.json()
+        if required_field not in json_data:
+            raise KeyError(f"'{required_field}' missing in response")
+        return json_data[required_field]
+
+    def generate_token_and_site_id(self, site_url: str) -> tuple[str, str]:
+        try:
+            parsed = urlparse(site_url)
+            domain = parsed.hostname
+            site_path = parsed.path.strip('/')
+            if not domain or not site_path:
+                raise ValueError(f"site_url missing domain or site path: {site_url}")
+            app_name = domain.split('.')[0]
+            openid_config_url = f"https://login.microsoftonline.com/{app_name}.onmicrosoft.com/v2.0/.well-known/openid-configuration"
+            response = requests.get(openid_config_url)
+            token_url = self._validate_response(response, required_field="token_endpoint", error_prefix="OpenID config")
+            token_data = {
+                "grant_type": "client_credentials",
+                "client_id": self.client_id,
+                "client_secret": self.client_secret,
+                "scope": "https://graph.microsoft.com/.default"
+            }
+            token_response = requests.post(token_url, data=token_data)
+            access_token = self._validate_response(token_response, required_field="access_token", error_prefix="Token request")
+            graph_site_url = f"https://graph.microsoft.com/v1.0/sites/{domain}:/{site_path}"
+            headers = {"Authorization": f"Bearer {access_token}"}
+            site_response = requests.get(graph_site_url, headers=headers)
+            site_id = self._validate_response(site_response, required_field="id", error_prefix="Site info")
+            return access_token, site_id
+        except Exception as e:
+            raise RuntimeError(f"Error while obtaining access_token and site_id: {e}")
+
+    def get_files_list(self, site_url: str, folder_name: str = None, limit_files: int = 100):
+        if not site_url or not site_url.startswith("https://"):
+            raise ValueError(f"Invalid site_url format: {site_url}")
+        if limit_files is not None and (not isinstance(limit_files, int) or limit_files <= 0):
+            raise ValueError(f"limit_files must be a positive integer, got: {limit_files}")
+        try:
+            access_token, site_id = self.generate_token_and_site_id(site_url)
+            headers = {"Authorization": f"Bearer {access_token}"}
+            drives_url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives"
+            drives_response = requests.get(drives_url, headers=headers)
+            drives = self._validate_response(drives_response, required_field="value", error_prefix="Drives request")
+            result = []
+            def _recurse_drive(drive_id, drive_path, parent_folder, limit_files):
+                # Escape folder_name for URL safety if present
+                if parent_folder:
+                    safe_folder_name = quote(parent_folder.strip('/'), safe="/")
+                    url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives/{drive_id}/root:/{safe_folder_name}:/children?$top={limit_files}"
+                else:
+                    url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives/{drive_id}/root/children?$top={limit_files}"
+                response = requests.get(url, headers=headers)
+                if response.status_code != 200:
+                    return []
+                files_json = response.json()
+                if "value" not in files_json:
+                    return []
+                files = []
+                for file in files_json["value"]:
+                    file_name = file.get('name', '')
+                    # Build full path reflecting nested folders
+                    if parent_folder:
+                        full_path = '/' + '/'.join([drive_path.strip('/'), parent_folder.strip('/'), file_name.strip('/')])
+                    else:
+                        full_path = '/' + '/'.join([drive_path.strip('/'), file_name.strip('/')])
+                    temp_props = {
+                        'Name': file_name,
+                        'Path': full_path,
+                        'Created': file.get('createdDateTime'),
+                        'Modified': file.get('lastModifiedDateTime'),
+                        'Link': file.get('webUrl'),
+                        'id': file.get('id')
+                    }
+                    if not all([temp_props['Name'], temp_props['Path'], temp_props['id']]):
+                        continue  # skip files with missing required fields
+                    if 'folder' in file:
+                        # Recursively extract files from this folder
+                        inner_folder = parent_folder + '/' + file_name if parent_folder else file_name
+                        inner_files = _recurse_drive(drive_id, drive_path, inner_folder, limit_files)
+                        files.extend(inner_files)
+                    else:
+                        files.append(temp_props)
+                    if limit_files is not None and len(result) + len(files) >= limit_files:
+                        return files[:limit_files - len(result)]
+                return files
+            for drive in drives:
+                drive_id = drive.get("id")
+                drive_path = unquote(urlparse(drive.get("webUrl")).path) if drive.get("webUrl") else ""
+                if not drive_id:
+                    continue  # skip drives without id
+                files = _recurse_drive(drive_id, drive_path, folder_name, limit_files)
+                result.extend(files)
+                if limit_files is not None and len(result) >= limit_files:
+                    return result[:limit_files]
+            return result
+        except Exception as e:
+            raise RuntimeError(f"Error in get_files_list: {e}")
+
+    def get_file_content(self, site_url: str, path: str):
+        try:
+            access_token, site_id = self.generate_token_and_site_id(site_url)
+            headers = {"Authorization": f"Bearer {access_token}"}
+            drives_url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/drives"
+            drives_response = requests.get(drives_url, headers=headers)
+            drives = self._validate_response(drives_response, required_field="value", error_prefix="Drives request")
+            path = path.strip('/')
+            #
+            for drive in drives:
+                drive_path = unquote(urlparse(drive.get("webUrl")).path).strip('/')
+                if not drive_path or not path.startswith(drive_path):
+                    continue
+                drive_id = drive.get("id")
+                if not drive_id:
+                    continue
+                path = path.replace(drive_path, '').strip('/')
+                safe_path = quote(path, safe="")
+                url = f"https://graph.microsoft.com/v1.0/drives/{drive_id}/root:/{safe_path}:/content"
+                response = requests.get(url, headers=headers)
+                if response.status_code == 200:
+                    return response.content
+            raise RuntimeError(f"File '{path}' not found in any private or shared documents.")
+        except Exception as e:
+            raise RuntimeError(f"Error in get_file_content: {e}")
+
+    def get_list_items(self, site_url: str, list_title: str, limit: int = 1000):
+        """Fallback Graph API method to read SharePoint list items by list title.
+
+        Returns a list of dictionaries representing list item fields.
+        """
+        if not site_url or not site_url.startswith("https://"):
+            raise ValueError(f"Invalid site_url format: {site_url}")
+        try:
+            access_token, site_id = self.generate_token_and_site_id(site_url)
+            headers = {"Authorization": f"Bearer {access_token}"}
+            lists_url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/lists"
+            response = requests.get(lists_url, headers=headers)
+            if response.status_code != 200:
+                raise RuntimeError(f"Lists request failed: {response.status_code} {response.text}")
+            lists_json = response.json()
+            lists = lists_json.get("value", [])
+            target_list = None
+            normalized_title = list_title.strip().lower()
+            for lst in lists:
+                # displayName is the user-visible title. name can differ (internal name)
+                display_name = (lst.get("displayName") or lst.get("name") or '').strip().lower()
+                if display_name == normalized_title:
+                    target_list = lst
+                    break
+            if not target_list:
+                raise RuntimeError(f"List '{list_title}' not found via Graph API.")
+            list_id = target_list.get('id')
+            if not list_id:
+                raise RuntimeError(f"List '{list_title}' missing id field.")
+            items_url = f"https://graph.microsoft.com/v1.0/sites/{site_id}/lists/{list_id}/items?expand=fields&$top={limit}"
+            items_response = requests.get(items_url, headers=headers)
+            if items_response.status_code != 200:
+                raise RuntimeError(f"List items request failed: {items_response.status_code} {items_response.text}")
+            items_json = items_response.json()
+            values = items_json.get('value', [])
+            result = []
+            for item in values:
+                fields = item.get('fields', {})
+                if fields:
+                    result.append(fields)
+            return result
+        except Exception as e:
+            raise RuntimeError(f"Error in get_list_items: {e}")

alita_sdk/tools/sharepoint/utils.py

@@ -1,5 +1,7 @@
-from docx import Document
+import re
 from io import BytesIO
+from docx import Document
+
 
 def read_docx_from_bytes(file_content):
     """Read and return content from a .docx file using a byte stream."""
@@ -11,4 +13,8 @@ def read_docx_from_bytes(file_content):
         return '\n'.join(text)
     except Exception as e:
         print(f"Error reading .docx from bytes: {e}")
-        return ""
+        return ""
+
+
+def decode_sharepoint_string(s):
+    return re.sub(r'_x([0-9A-Fa-f]{4})_', lambda m: chr(int(m.group(1), 16)), s)

alita_sdk/tools/utils/content_parser.py

@@ -92,21 +92,24 @@ def parse_file_content(file_name=None, file_content=None, is_capture_image: bool
         return ToolException(
             "Not supported type of files entered. Supported types are TXT, DOCX, PDF, PPTX, XLSX and XLS only.")
 
-    if hasattr(loader, 'get_content'):
-        return loader.get_content()
-    else:
-        extension = Path(file_path if file_path else file_name).suffix
-        loader_kwargs = get_loader_kwargs(loaders_map.get(extension), file_name, file_content, is_capture_image, page_number, sheet_name, llm, file_path, excel_by_sheets)
-        if file_content:
-            return load_content_from_bytes(file_content=file_content,
-                                           extension=extension,
-                                           loader_extra_config=loader_kwargs,
-                                           llm=llm)
+    try:
+        if hasattr(loader, 'get_content'):
+            return loader.get_content()
         else:
-            return load_content(file_path=file_path,
-                                extension=extension,
-                                loader_extra_config=loader_kwargs,
-                                llm=llm)
+            extension = Path(file_path if file_path else file_name).suffix
+            loader_kwargs = get_loader_kwargs(loaders_map.get(extension), file_name, file_content, is_capture_image, page_number, sheet_name, llm, file_path, excel_by_sheets)
+            if file_content:
+                return load_content_from_bytes(file_content=file_content,
+                                               extension=extension,
+                                               loader_extra_config=loader_kwargs,
+                                               llm=llm)
+            else:
+                return load_content(file_path=file_path,
+                                    extension=extension,
+                                    loader_extra_config=loader_kwargs,
+                                    llm=llm)
+    except Exception as e:
+        return ToolException(f"Error reading file ({file_name or file_path}) content. Make sure these types are supported: {str(e)}")
 
 def load_file_docs(file_name=None, file_content=None, is_capture_image: bool = False, page_number: int = None,
                        sheet_name: str = None, llm=None, file_path: str = None, excel_by_sheets: bool = False) -> List[Document] | ToolException:
@@ -153,7 +156,7 @@ def prepare_loader(file_name=None, file_content=None, is_capture_image: bool = F
 
         loader_object = loaders_map.get(extension)
         if not loader_object:
-            return None
+            loader_object = loaders_map.get('.txt')  # Default to text loader if no specific loader found
         loader_kwargs = get_loader_kwargs(loader_object, file_name, file_content, is_capture_image, page_number, sheet_name, llm, file_path, excel_by_sheets, prompt)
         loader = loader_object['class'](**loader_kwargs)
         return loader
@@ -222,10 +225,18 @@ def process_document_by_type(content, extension_source: str, document: Document
             metadata={**document.metadata, 'chunk_id': 1}
         )
         return
+    #
+    chunks_counter = 0
     for chunk in chunks:
+        chunks_counter += 1
+        metadata = {**document.metadata, **chunk.metadata}
+        #
+        # ensure each chunk has a unique chunk_id
+        metadata['chunk_id'] = chunks_counter
+        #
         yield Document(
             page_content=sanitize_for_postgres(chunk.page_content),
-            metadata={**document.metadata, **chunk.metadata}
+            metadata=metadata
         )
 
 

alita_sdk/tools/vector_adapters/VectorStoreAdapter.py

@@ -26,12 +26,12 @@ class VectorStoreAdapter(ABC):
         pass
 
     @abstractmethod
-    def get_indexed_ids(self, vectorstore_wrapper, collection_suffix: Optional[str] = '') -> List[str]:
+    def get_indexed_ids(self, vectorstore_wrapper, index_name: Optional[str] = '') -> List[str]:
         """Get all indexed document IDs from vectorstore"""
         pass
 
     @abstractmethod
-    def clean_collection(self, vectorstore_wrapper, collection_suffix: str = ''):
+    def clean_collection(self, vectorstore_wrapper, index_name: str = ''):
         """Clean the vectorstore collection by deleting all indexed data."""
         pass
 
@@ -41,7 +41,7 @@ class VectorStoreAdapter(ABC):
         pass
 
     @abstractmethod
-    def get_code_indexed_data(self, vectorstore_wrapper, collection_suffix) -> Dict[str, Dict[str, Any]]:
+    def get_code_indexed_data(self, vectorstore_wrapper, index_name) -> Dict[str, Dict[str, Any]]:
         """Get all indexed data from vectorstore for code content"""
         pass
 
@@ -51,7 +51,7 @@ class VectorStoreAdapter(ABC):
         pass
 
     @abstractmethod
-    def get_index_meta(self, vectorstore_wrapper, collection_suffix: str) -> List[Dict[str, Any]]:
+    def get_index_meta(self, vectorstore_wrapper, index_name: str) -> List[Dict[str, Any]]:
         """Get all index_meta entries from the vector store."""
         pass
 
@@ -106,20 +106,25 @@ class PGVectorAdapter(VectorStoreAdapter):
             session.commit()
             logger.info(f"Schema '{schema_name}' has been dropped.")
 
-    def get_indexed_ids(self, vectorstore_wrapper, collection_suffix: Optional[str] = '') -> List[str]:
+    def get_indexed_ids(self, vectorstore_wrapper, index_name: Optional[str] = '') -> List[str]:
         """Get all indexed document IDs from PGVector"""
         from sqlalchemy.orm import Session
-        from sqlalchemy import func
+        from sqlalchemy import func, or_
 
         store = vectorstore_wrapper.vectorstore
         try:
             with Session(store.session_maker.bind) as session:
                 # Start building the query
                 query = session.query(store.EmbeddingStore.id)
-                # Apply filter only if collection_suffix is provided
-                if collection_suffix:
+                # Apply filter only if index_name is provided
+                if index_name:
                     query = query.filter(
-                        func.jsonb_extract_path_text(store.EmbeddingStore.cmetadata, 'collection') == collection_suffix
+                        func.jsonb_extract_path_text(store.EmbeddingStore.cmetadata, 'collection') == index_name,
+                        or_(
+                            func.jsonb_extract_path_text(store.EmbeddingStore.cmetadata, 'type').is_(None),
+                            func.jsonb_extract_path_text(store.EmbeddingStore.cmetadata,
+                                                         'type') != IndexerKeywords.INDEX_META_TYPE.value
+                        )
                     )
                 ids = query.all()
                 return [str(id_tuple[0]) for id_tuple in ids]
@@ -127,25 +132,33 @@ class PGVectorAdapter(VectorStoreAdapter):
             logger.error(f"Failed to get indexed IDs from PGVector: {str(e)}")
             return []
 
-    def clean_collection(self, vectorstore_wrapper, collection_suffix: str = ''):
+    def clean_collection(self, vectorstore_wrapper, index_name: str = ''):
         """Clean the vectorstore collection by deleting all indexed data."""
         # This logic deletes all data from the vectorstore collection without removal of collection.
         # Collection itself remains available for future indexing.
-        vectorstore_wrapper.vectorstore.delete(ids=self.get_indexed_ids(vectorstore_wrapper, collection_suffix))
+        from sqlalchemy.orm import Session
+        from sqlalchemy import func
+
+        store = vectorstore_wrapper.vectorstore
+        with Session(store.session_maker.bind) as session:
+            session.query(store.EmbeddingStore).filter(
+                func.jsonb_extract_path_text(store.EmbeddingStore.cmetadata, 'collection') == index_name
+            ).delete(synchronize_session=False)
+            session.commit()
 
     def is_vectorstore_type(self, vectorstore) -> bool:
         """Check if the vectorstore is a PGVector store."""
         return hasattr(vectorstore, 'session_maker') and hasattr(vectorstore, 'EmbeddingStore')
 
-    def get_indexed_data(self, vectorstore_wrapper, collection_suffix: str)-> Dict[str, Dict[str, Any]]:
-        """Get all indexed data from PGVector for non-code content per collection_suffix."""
+    def get_indexed_data(self, vectorstore_wrapper, index_name: str)-> Dict[str, Dict[str, Any]]:
+        """Get all indexed data from PGVector for non-code content per index_name."""
         from sqlalchemy.orm import Session
         from sqlalchemy import func
         from ...runtime.utils.utils import IndexerKeywords
 
         result = {}
         try:
-            vectorstore_wrapper._log_data("Retrieving already indexed data from PGVector vectorstore",
+            vectorstore_wrapper._log_tool_event("Retrieving already indexed data from PGVector vectorstore",
                            tool_name="get_indexed_data")
             store = vectorstore_wrapper.vectorstore
             with Session(store.session_maker.bind) as session:
@@ -154,7 +167,7 @@ class PGVectorAdapter(VectorStoreAdapter):
                     store.EmbeddingStore.document,
                     store.EmbeddingStore.cmetadata
                 ).filter(
-                    func.jsonb_extract_path_text(store.EmbeddingStore.cmetadata, 'collection') == collection_suffix
+                    func.jsonb_extract_path_text(store.EmbeddingStore.cmetadata, 'collection') == index_name
                 ).all()
 
             # Process the retrieved data
@@ -187,14 +200,14 @@ class PGVectorAdapter(VectorStoreAdapter):
 
         return result
 
-    def get_code_indexed_data(self, vectorstore_wrapper, collection_suffix: str) -> Dict[str, Dict[str, Any]]:
+    def get_code_indexed_data(self, vectorstore_wrapper, index_name: str) -> Dict[str, Dict[str, Any]]:
         """Get all indexed code data from PGVector per collection suffix."""
         from sqlalchemy.orm import Session
         from sqlalchemy import func
 
         result = {}
         try:
-            vectorstore_wrapper._log_data("Retrieving already indexed code data from PGVector vectorstore",
+            vectorstore_wrapper._log_tool_event(message="Retrieving already indexed code data from PGVector vectorstore",
                            tool_name="index_code_data")
             store = vectorstore_wrapper.vectorstore
             with (Session(store.session_maker.bind) as session):
@@ -202,7 +215,7 @@ class PGVectorAdapter(VectorStoreAdapter):
                     store.EmbeddingStore.id,
                     store.EmbeddingStore.cmetadata
                 ).filter(
-                    func.jsonb_extract_path_text(store.EmbeddingStore.cmetadata, 'collection') == collection_suffix
+                    func.jsonb_extract_path_text(store.EmbeddingStore.cmetadata, 'collection') == index_name
                 ).all()
 
             for db_id, meta in docs:
@@ -272,7 +285,7 @@ class PGVectorAdapter(VectorStoreAdapter):
         except Exception as e:
             logger.error(f"Failed to update collection for entry ID {entry_id}: {str(e)}")
 
-    def get_index_meta(self, vectorstore_wrapper, collection_suffix: str) -> List[Dict[str, Any]]:
+    def get_index_meta(self, vectorstore_wrapper, index_name: str) -> List[Dict[str, Any]]:
         from sqlalchemy.orm import Session
         from sqlalchemy import func
 
@@ -285,7 +298,7 @@ class PGVectorAdapter(VectorStoreAdapter):
                     store.EmbeddingStore.cmetadata
                 ).filter(
                     store.EmbeddingStore.cmetadata['type'].astext == IndexerKeywords.INDEX_META_TYPE.value,
-                    func.jsonb_extract_path_text(store.EmbeddingStore.cmetadata, 'collection') == collection_suffix
+                    func.jsonb_extract_path_text(store.EmbeddingStore.cmetadata, 'collection') == index_name
                 ).all()
                 result = []
                 for id, document, cmetadata in meta:
@@ -312,7 +325,7 @@ class ChromaAdapter(VectorStoreAdapter):
     def remove_collection(self, vectorstore_wrapper, collection_name: str):
         vectorstore_wrapper.vectorstore.delete_collection()
 
-    def get_indexed_ids(self, vectorstore_wrapper, collection_suffix: Optional[str] = '') -> List[str]:
+    def get_indexed_ids(self, vectorstore_wrapper, index_name: Optional[str] = '') -> List[str]:
         """Get all indexed document IDs from Chroma"""
         try:
             data = vectorstore_wrapper.vectorstore.get(include=[])  # Only get IDs, no metadata
@@ -321,9 +334,9 @@ class ChromaAdapter(VectorStoreAdapter):
             logger.error(f"Failed to get indexed IDs from Chroma: {str(e)}")
             return []
 
-    def clean_collection(self, vectorstore_wrapper, collection_suffix: str = ''):
+    def clean_collection(self, vectorstore_wrapper, index_name: str = ''):
         """Clean the vectorstore collection by deleting all indexed data."""
-        vectorstore_wrapper.vectorstore.delete(ids=self.get_indexed_ids(vectorstore_wrapper, collection_suffix))
+        vectorstore_wrapper.vectorstore.delete(ids=self.get_indexed_ids(vectorstore_wrapper, index_name))
 
     def get_indexed_data(self, vectorstore_wrapper):
         """Get all indexed data from Chroma for non-code content"""
@@ -361,7 +374,7 @@ class ChromaAdapter(VectorStoreAdapter):
 
         return result
 
-    def get_code_indexed_data(self, vectorstore_wrapper, collection_suffix) -> Dict[str, Dict[str, Any]]:
+    def get_code_indexed_data(self, vectorstore_wrapper, index_name) -> Dict[str, Dict[str, Any]]:
         """Get all indexed code data from Chroma."""
         result = {}
         try:
@@ -391,7 +404,7 @@ class ChromaAdapter(VectorStoreAdapter):
         # This is a simplified implementation - in practice, you might need more complex logic
         logger.warning("add_to_collection for Chroma is not fully implemented yet")
 
-    def get_index_meta(self, vectorstore_wrapper, collection_suffix: str) -> List[Dict[str, Any]]:
+    def get_index_meta(self, vectorstore_wrapper, index_name: str) -> List[Dict[str, Any]]:
         logger.warning("get_index_meta for Chroma is not implemented yet")
 
 

{alita_sdk-0.3.374.dist-info → alita_sdk-0.3.423.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.374
+Version: 0.3.423
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedj27@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0