sonika-langchain-bot 0.0.13__tar.gz → 0.0.14__tar.gz

{sonika_langchain_bot-0.0.13/src/sonika_langchain_bot.egg-info → sonika_langchain_bot-0.0.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sonika-langchain-bot
-Version: 0.0.13
+Version: 0.0.14
 Summary: Agente langchain con LLM
 Author: Erley Blanco Carvajal
 License: MIT License
@@ -25,6 +25,10 @@ Requires-Dist: pypdf==5.6.1
 Requires-Dist: python-dotenv==1.0.1
 Requires-Dist: typing_extensions==4.14.0
 Requires-Dist: typing-inspect==0.9.0
+Requires-Dist: PyPDF2==3.0.1
+Requires-Dist: python-docx==1.2.0
+Requires-Dist: openpyxl==3.1.5
+Requires-Dist: python-pptx==1.0.2
 Provides-Extra: dev
 Requires-Dist: sphinx<9.0.0,>=8.1.3; extra == "dev"
 Requires-Dist: sphinx-rtd-theme<4.0.0,>=3.0.1; extra == "dev"

{sonika_langchain_bot-0.0.13 → sonika_langchain_bot-0.0.14}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="sonika-langchain-bot",
-    version="0.0.13", 
+    version="0.0.14", 
     description="Agente langchain con LLM",
     author="Erley Blanco Carvajal",
     license="MIT License",
@@ -26,6 +26,10 @@ setup(
         "python-dotenv==1.0.1",
         "typing_extensions==4.14.0",
         "typing-inspect==0.9.0",
+        "PyPDF2==3.0.1",
+        "python-docx==1.2.0",
+        "openpyxl==3.1.5",
+        "python-pptx==1.0.2"
     ],
 
     extras_require={

sonika_langchain_bot-0.0.14/src/sonika_langchain_bot/document_processor.py

@@ -0,0 +1,334 @@
+"""
+Document processing utilities for text extraction and chunking.
+
+This module provides tools to extract text from various document formats
+and split them into manageable chunks for processing.
+"""
+
+import tiktoken
+from typing import List, Dict, Optional
+
+
+class DocumentProcessor:
+    """
+    Service for processing documents and generating text chunks.
+    
+    Supports extraction from PDF, DOCX, TXT, XLSX, PPTX and other formats.
+    Provides intelligent text chunking with configurable overlap.
+    """
+    
+    @staticmethod
+    def count_tokens(text: str, model: str = "gpt-4") -> int:
+        """
+        Count tokens in text using tiktoken encoding.
+        
+        Args:
+            text: Text to count tokens from
+            model: Model name for encoding reference (default: "gpt-4")
+            
+        Returns:
+            int: Number of tokens in the text
+            
+        Note:
+            Falls back to character-based approximation (1 token ≈ 4 chars)
+            if tiktoken encoding fails.
+        """
+        try:
+            encoding = tiktoken.encoding_for_model(model)
+            return len(encoding.encode(text))
+        except Exception:
+            # Fallback: approximation (1 token ≈ 4 characters)
+            return len(text) // 4
+    
+    @staticmethod
+    def extract_text_from_pdf(file_path: str) -> str:
+        """
+        Extract text content from PDF file.
+        
+        Args:
+            file_path: Path to the PDF file
+            
+        Returns:
+            str: Extracted text with page markers
+            
+        Raises:
+            Exception: If PDF extraction fails
+        """
+        try:
+            import PyPDF2
+        except ImportError:
+            raise ImportError(
+                "PyPDF2 is required for PDF processing. "
+                "Install with: pip install sonika-langchain-bot[documents]"
+            )
+        
+        text = ""
+        try:
+            with open(file_path, 'rb') as file:
+                pdf_reader = PyPDF2.PdfReader(file)
+                for page_num, page in enumerate(pdf_reader.pages):
+                    page_text = page.extract_text()
+                    if page_text:
+                        text += f"\n--- Página {page_num + 1} ---\n{page_text}\n"
+        except Exception as e:
+            raise Exception(f"Error extracting text from PDF: {str(e)}")
+        return text.strip()
+    
+    @staticmethod
+    def extract_text_from_docx(file_path: str) -> str:
+        """
+        Extract text content from DOCX file.
+        
+        Args:
+            file_path: Path to the DOCX file
+            
+        Returns:
+            str: Extracted text
+            
+        Raises:
+            Exception: If DOCX extraction fails
+        """
+        try:
+            import docx
+        except ImportError:
+            raise ImportError(
+                "python-docx is required for DOCX processing. "
+                "Install with: pip install sonika-langchain-bot[documents]"
+            )
+        
+        try:
+            doc = docx.Document(file_path)
+            text = "\n".join([
+                paragraph.text 
+                for paragraph in doc.paragraphs 
+                if paragraph.text.strip()
+            ])
+            return text.strip()
+        except Exception as e:
+            raise Exception(f"Error extracting text from DOCX: {str(e)}")
+    
+    @staticmethod
+    def extract_text_from_txt(file_path: str) -> str:
+        """
+        Extract text content from plain text file.
+        
+        Args:
+            file_path: Path to the text file
+            
+        Returns:
+            str: File content
+            
+        Note:
+            Attempts UTF-8 encoding first, falls back to latin-1
+        """
+        try:
+            with open(file_path, 'r', encoding='utf-8') as file:
+                return file.read().strip()
+        except UnicodeDecodeError:
+            # Fallback to latin-1 if UTF-8 fails
+            with open(file_path, 'r', encoding='latin-1') as file:
+                return file.read().strip()
+    
+    @staticmethod
+    def extract_text_from_xlsx(file_path: str) -> str:
+        """
+        Extract text content from Excel file.
+        
+        Args:
+            file_path: Path to the Excel file
+            
+        Returns:
+            str: Extracted text with sheet and row markers
+            
+        Raises:
+            Exception: If Excel extraction fails
+        """
+        try:
+            from openpyxl import load_workbook
+        except ImportError:
+            raise ImportError(
+                "openpyxl is required for Excel processing. "
+                "Install with: pip install sonika-langchain-bot[documents]"
+            )
+        
+        try:
+            workbook = load_workbook(file_path, data_only=True)
+            text = ""
+            
+            for sheet_name in workbook.sheetnames:
+                sheet = workbook[sheet_name]
+                text += f"\n--- Hoja: {sheet_name} ---\n"
+                
+                for row in sheet.iter_rows(values_only=True):
+                    row_text = " | ".join([
+                        str(cell) if cell is not None else "" 
+                        for cell in row
+                    ])
+                    if row_text.strip():
+                        text += row_text + "\n"
+            
+            return text.strip()
+        except Exception as e:
+            raise Exception(f"Error extracting text from Excel: {str(e)}")
+    
+    @staticmethod
+    def extract_text_from_pptx(file_path: str) -> str:
+        """
+        Extract text content from PowerPoint file.
+        
+        Args:
+            file_path: Path to the PowerPoint file
+            
+        Returns:
+            str: Extracted text with slide markers
+            
+        Raises:
+            Exception: If PowerPoint extraction fails
+        """
+        try:
+            from pptx import Presentation
+        except ImportError:
+            raise ImportError(
+                "python-pptx is required for PowerPoint processing. "
+                "Install with: pip install sonika-langchain-bot[documents]"
+            )
+        
+        try:
+            prs = Presentation(file_path)
+            text = ""
+            
+            for slide_num, slide in enumerate(prs.slides, 1):
+                text += f"\n--- Diapositiva {slide_num} ---\n"
+                for shape in slide.shapes:
+                    if hasattr(shape, "text") and shape.text.strip():
+                        text += shape.text + "\n"
+            
+            return text.strip()
+        except Exception as e:
+            raise Exception(f"Error extracting text from PowerPoint: {str(e)}")
+    
+    @classmethod
+    def extract_text(cls, file_path: str, file_extension: str) -> str:
+        """
+        Extract text from file based on extension.
+        
+        Args:
+            file_path: Path to the file
+            file_extension: File extension (without dot)
+            
+        Returns:
+            str: Extracted text
+            
+        Raises:
+            ValueError: If file format is not supported
+        """
+        extractors = {
+            'pdf': cls.extract_text_from_pdf,
+            'docx': cls.extract_text_from_docx,
+            'doc': cls.extract_text_from_docx,
+            'txt': cls.extract_text_from_txt,
+            'md': cls.extract_text_from_txt,
+            'xlsx': cls.extract_text_from_xlsx,
+            'xls': cls.extract_text_from_xlsx,
+            'csv': cls.extract_text_from_txt,
+            'pptx': cls.extract_text_from_pptx,
+            'ppt': cls.extract_text_from_pptx,
+        }
+        
+        extractor = extractors.get(file_extension.lower())
+        if not extractor:
+            supported = ', '.join(extractors.keys())
+            raise ValueError(
+                f"Format '{file_extension}' not supported. "
+                f"Supported formats: {supported}"
+            )
+        
+        return extractor(file_path)
+    
+    @classmethod
+    def create_chunks(
+        cls, 
+        text: str, 
+        chunk_size: int = 1000, 
+        overlap: int = 200
+    ) -> List[Dict]:
+        """
+        Split text into chunks with configurable overlap.
+        
+        Args:
+            text: Complete text to chunk
+            chunk_size: Maximum chunk size in tokens (default: 1000)
+            overlap: Token overlap between chunks (default: 200)
+            
+        Returns:
+            List[Dict]: List of chunks with metadata
+                Each chunk contains:
+                - content: Text content
+                - chunk_index: Sequential index
+                - token_count: Number of tokens
+                - metadata: Additional metadata (empty dict)
+                
+        Example:
+            >>> processor = DocumentProcessor()
+            >>> chunks = processor.create_chunks("Long text...", chunk_size=500)
+            >>> print(chunks[0])
+            {
+                'content': 'First chunk text...',
+                'chunk_index': 0,
+                'token_count': 450,
+                'metadata': {}
+            }
+        """
+        # Split into sentences (approximate)
+        sentences = text.replace('\n', ' ').split('. ')
+        
+        chunks = []
+        current_chunk = ""
+        current_tokens = 0
+        chunk_index = 0
+        
+        for sentence in sentences:
+            sentence = sentence.strip()
+            if not sentence:
+                continue
+            
+            sentence_tokens = cls.count_tokens(sentence)
+            
+            # Check if adding sentence exceeds chunk size
+            if current_tokens + sentence_tokens > chunk_size and current_chunk:
+                # Save current chunk
+                chunks.append({
+                    'content': current_chunk.strip(),
+                    'chunk_index': chunk_index,
+                    'token_count': current_tokens,
+                    'metadata': {}
+                })
+                
+                # Prepare next chunk with overlap
+                overlap_text = (
+                    ' '.join(current_chunk.split()[-overlap:]) 
+                    if overlap > 0 else ""
+                )
+                current_chunk = (
+                    overlap_text + " " + sentence 
+                    if overlap_text else sentence
+                )
+                current_tokens = cls.count_tokens(current_chunk)
+                chunk_index += 1
+            else:
+                # Add sentence to current chunk
+                current_chunk += (
+                    " " + sentence if current_chunk else sentence
+                )
+                current_tokens += sentence_tokens
+        
+        # Add last chunk if exists
+        if current_chunk.strip():
+            chunks.append({
+                'content': current_chunk.strip(),
+                'chunk_index': chunk_index,
+                'token_count': current_tokens,
+                'metadata': {}
+            })
+        
+        return chunks

{sonika_langchain_bot-0.0.13 → sonika_langchain_bot-0.0.14/src/sonika_langchain_bot.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sonika-langchain-bot
-Version: 0.0.13
+Version: 0.0.14
 Summary: Agente langchain con LLM
 Author: Erley Blanco Carvajal
 License: MIT License
@@ -25,6 +25,10 @@ Requires-Dist: pypdf==5.6.1
 Requires-Dist: python-dotenv==1.0.1
 Requires-Dist: typing_extensions==4.14.0
 Requires-Dist: typing-inspect==0.9.0
+Requires-Dist: PyPDF2==3.0.1
+Requires-Dist: python-docx==1.2.0
+Requires-Dist: openpyxl==3.1.5
+Requires-Dist: python-pptx==1.0.2
 Provides-Extra: dev
 Requires-Dist: sphinx<9.0.0,>=8.1.3; extra == "dev"
 Requires-Dist: sphinx-rtd-theme<4.0.0,>=3.0.1; extra == "dev"

{sonika_langchain_bot-0.0.13 → sonika_langchain_bot-0.0.14}/src/sonika_langchain_bot.egg-info/SOURCES.txt

@@ -2,6 +2,7 @@ LICENSE
 README.md
 setup.py
 src/sonika_langchain_bot/__init__.py
+src/sonika_langchain_bot/document_processor.py
 src/sonika_langchain_bot/langchain_bot_agent.py
 src/sonika_langchain_bot/langchain_clasificator.py
 src/sonika_langchain_bot/langchain_class.py
@@ -13,4 +14,5 @@ src/sonika_langchain_bot.egg-info/SOURCES.txt
 src/sonika_langchain_bot.egg-info/dependency_links.txt
 src/sonika_langchain_bot.egg-info/requires.txt
 src/sonika_langchain_bot.egg-info/top_level.txt
-test/test.py
+test/test.py
+test/test_document_processor.py

{sonika_langchain_bot-0.0.13 → sonika_langchain_bot-0.0.14}/src/sonika_langchain_bot.egg-info/requires.txt

@@ -13,6 +13,10 @@ pypdf==5.6.1
 python-dotenv==1.0.1
 typing_extensions==4.14.0
 typing-inspect==0.9.0
+PyPDF2==3.0.1
+python-docx==1.2.0
+openpyxl==3.1.5
+python-pptx==1.0.2
 
 [dev]
 sphinx<9.0.0,>=8.1.3

sonika_langchain_bot-0.0.14/test/test_document_processor.py

@@ -0,0 +1,222 @@
+# test_document_processor.py
+import os
+import sys
+from pathlib import Path
+
+# Añadir la carpeta 'src' al PYTHONPATH
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'src'))
+
+
+from sonika_langchain_bot.document_processor import DocumentProcessor
+
+
+def create_test_files():
+    """Crea archivos de prueba si no existen"""
+    test_dir = Path("test_documents")
+    test_dir.mkdir(exist_ok=True)
+    
+    # Crear archivo TXT de prueba
+    txt_file = test_dir / "test.txt"
+    if not txt_file.exists():
+        txt_file.write_text(
+            "Este es un documento de prueba.\n"
+            "Contiene múltiples líneas de texto.\n"
+            "Será usado para probar el DocumentProcessor.\n" * 10,
+            encoding='utf-8'
+        )
+    
+    return test_dir
+
+
+def test_token_counting():
+    """Prueba el conteo de tokens"""
+    print("\n" + "="*60)
+    print("TEST 1: Conteo de tokens")
+    print("="*60)
+    
+    test_text = "Este es un texto de prueba para contar tokens."
+    token_count = DocumentProcessor.count_tokens(test_text)
+    
+    print(f"Texto: {test_text}")
+    print(f"Tokens contados: {token_count}")
+    
+    assert token_count > 0, "El conteo de tokens debe ser mayor a 0"
+    print("✅ Test de conteo de tokens: PASSED")
+
+
+def test_txt_extraction():
+    """Prueba extracción de texto TXT"""
+    print("\n" + "="*60)
+    print("TEST 2: Extracción de texto TXT")
+    print("="*60)
+    
+    test_dir = create_test_files()
+    txt_file = test_dir / "test.txt"
+    
+    try:
+        text = DocumentProcessor.extract_text(str(txt_file), "txt")
+        print(f"Texto extraído ({len(text)} caracteres):")
+        print(text[:200] + "..." if len(text) > 200 else text)
+        
+        assert len(text) > 0, "El texto extraído no debe estar vacío"
+        assert "documento de prueba" in text.lower(), "El texto debe contener el contenido esperado"
+        print("✅ Test de extracción TXT: PASSED")
+        
+        return text
+    except Exception as e:
+        print(f"❌ Test de extracción TXT: FAILED - {str(e)}")
+        raise
+
+
+def test_chunking(text):
+    """Prueba la creación de chunks"""
+    print("\n" + "="*60)
+    print("TEST 3: Creación de chunks")
+    print("="*60)
+    
+    try:
+        chunks = DocumentProcessor.create_chunks(
+            text=text,
+            chunk_size=100,  # Más pequeño para testing
+            overlap=20
+        )
+        
+        print(f"Número de chunks generados: {len(chunks)}")
+        
+        assert len(chunks) > 0, "Debe generar al menos un chunk"
+        
+        # Verificar estructura de cada chunk
+        for i, chunk in enumerate(chunks[:3]):  # Mostrar solo primeros 3
+            print(f"\n--- Chunk {i} ---")
+            print(f"Index: {chunk['chunk_index']}")
+            print(f"Tokens: {chunk['token_count']}")
+            print(f"Content: {chunk['content'][:100]}...")
+            
+            assert 'content' in chunk, "Chunk debe tener 'content'"
+            assert 'chunk_index' in chunk, "Chunk debe tener 'chunk_index'"
+            assert 'token_count' in chunk, "Chunk debe tener 'token_count'"
+            assert 'metadata' in chunk, "Chunk debe tener 'metadata'"
+            assert chunk['chunk_index'] == i, "Los índices deben ser secuenciales"
+        
+        print(f"\n✅ Test de chunking: PASSED ({len(chunks)} chunks generados)")
+        
+        return chunks
+    except Exception as e:
+        print(f"❌ Test de chunking: FAILED - {str(e)}")
+        raise
+
+
+def test_unsupported_format():
+    """Prueba manejo de formato no soportado"""
+    print("\n" + "="*60)
+    print("TEST 4: Formato no soportado")
+    print("="*60)
+    
+    try:
+        DocumentProcessor.extract_text("test.xyz", "xyz")
+        print("❌ Test de formato no soportado: FAILED - Debería haber lanzado ValueError")
+        assert False, "Debería haber lanzado ValueError"
+    except ValueError as e:
+        print(f"Error esperado capturado: {str(e)}")
+        assert "not supported" in str(e).lower(), "El mensaje de error debe indicar formato no soportado"
+        print("✅ Test de formato no soportado: PASSED")
+
+
+def test_pdf_extraction_optional():
+    """Prueba extracción de PDF si existe"""
+    print("\n" + "="*60)
+    print("TEST 5: Extracción de PDF (opcional)")
+    print("="*60)
+    
+    test_pdf = "test_documents/sample.pdf"
+    
+    if not os.path.exists(test_pdf):
+        print(f"⚠️  No se encontró {test_pdf}")
+        print("Para probar PDF, coloca un archivo PDF en test_documents/sample.pdf")
+        print("✅ Test de PDF: SKIPPED")
+        return
+    
+    try:
+        text = DocumentProcessor.extract_text(test_pdf, "pdf")
+        print(f"Texto extraído de PDF ({len(text)} caracteres):")
+        print(text[:200] + "..." if len(text) > 200 else text)
+        
+        assert len(text) > 0, "El texto extraído del PDF no debe estar vacío"
+        print("✅ Test de extracción PDF: PASSED")
+    except ImportError as e:
+        print(f"⚠️  PyPDF2 no instalado: {str(e)}")
+        print("Instala con: pip install PyPDF2")
+        print("✅ Test de PDF: SKIPPED")
+    except Exception as e:
+        print(f"❌ Test de extracción PDF: FAILED - {str(e)}")
+
+
+def test_docx_extraction_optional():
+    """Prueba extracción de DOCX si existe"""
+    print("\n" + "="*60)
+    print("TEST 6: Extracción de DOCX (opcional)")
+    print("="*60)
+    
+    test_docx = "test_documents/sample.docx"
+    
+    if not os.path.exists(test_docx):
+        print(f"⚠️  No se encontró {test_docx}")
+        print("Para probar DOCX, coloca un archivo DOCX en test_documents/sample.docx")
+        print("✅ Test de DOCX: SKIPPED")
+        return
+    
+    try:
+        text = DocumentProcessor.extract_text(test_docx, "docx")
+        print(f"Texto extraído de DOCX ({len(text)} caracteres):")
+        print(text[:200] + "..." if len(text) > 200 else text)
+        
+        assert len(text) > 0, "El texto extraído del DOCX no debe estar vacío"
+        print("✅ Test de extracción DOCX: PASSED")
+    except ImportError as e:
+        print(f"⚠️  python-docx no instalado: {str(e)}")
+        print("Instala con: pip install python-docx")
+        print("✅ Test de DOCX: SKIPPED")
+    except Exception as e:
+        print(f"❌ Test de extracción DOCX: FAILED - {str(e)}")
+
+
+def run_all_tests():
+    """Ejecuta todos los tests"""
+    print("\n" + "🧪" * 30)
+    print("INICIANDO TESTS DE DocumentProcessor")
+    print("🧪" * 30)
+    
+    try:
+        # Tests obligatorios
+        test_token_counting()
+        text = test_txt_extraction()
+        test_chunking(text)
+        test_unsupported_format()
+        
+        # Tests opcionales (si hay archivos)
+        test_pdf_extraction_optional()
+        test_docx_extraction_optional()
+        
+        # Resumen
+        print("\n" + "="*60)
+        print("RESUMEN DE TESTS")
+        print("="*60)
+        print("✅ Todos los tests obligatorios: PASSED")
+        print("\nPara probar más formatos:")
+        print("1. Coloca un PDF en: test_documents/sample.pdf")
+        print("2. Coloca un DOCX en: test_documents/sample.docx")
+        print("3. Ejecuta de nuevo este script")
+        
+        return True
+        
+    except Exception as e:
+        print("\n" + "="*60)
+        print("❌ TESTS FALLIDOS")
+        print("="*60)
+        print(f"Error: {str(e)}")
+        return False
+
+
+if __name__ == "__main__":
+    success = run_all_tests()
+    sys.exit(0 if success else 1)