iatoolkit 0.3.9__py3-none-any.whl → 0.107.4__py3-none-any.whl

iatoolkit/services/dispatcher_service.py

@@ -0,0 +1,134 @@
+# Copyright (c) 2024 Fernando Libedinsky
+# Product: IAToolkit
+#
+# IAToolkit is open source software.
+
+from iatoolkit.common.exceptions import IAToolkitException
+from iatoolkit.services.prompt_service import PromptService
+from iatoolkit.repositories.llm_query_repo import LLMQueryRepo
+from iatoolkit.services.configuration_service import ConfigurationService
+from iatoolkit.common.util import Utility
+from injector import inject
+import logging
+import os
+
+
+class Dispatcher:
+    @inject
+    def __init__(self,
+                 config_service: ConfigurationService,
+                 prompt_service: PromptService,
+                 llmquery_repo: LLMQueryRepo,
+                 util: Utility,):
+        self.config_service = config_service
+        self.prompt_service = prompt_service
+        self.llmquery_repo = llmquery_repo
+        self.util = util
+
+        self._tool_service = None
+        self._company_registry = None
+        self._company_instances = None
+
+
+    @property
+    def tool_service(self):
+        """Lazy-loads and returns the ToolService instance to avoid circular imports."""
+        if self._tool_service is None:
+            from iatoolkit import current_iatoolkit
+            from iatoolkit.services.tool_service import ToolService
+            self._tool_service = current_iatoolkit().get_injector().get(ToolService)
+        return self._tool_service
+
+    @property
+    def company_registry(self):
+        """Lazy-loads and returns the CompanyRegistry instance."""
+        if self._company_registry is None:
+            from iatoolkit.company_registry import get_company_registry
+            self._company_registry = get_company_registry()
+        return self._company_registry
+
+    @property
+    def company_instances(self):
+        """Lazy-loads and returns the instantiated company classes."""
+        if self._company_instances is None:
+            self._company_instances = self.company_registry.get_all_company_instances()
+        return self._company_instances
+
+    def load_company_configs(self):
+        # initialize the system functions and prompts
+        self.setup_iatoolkit_system()
+
+        # Loads the configuration of every company: company.yaml file
+        for company_short_name, company_instance in self.company_instances.items():
+            try:
+                # read company configuration from company.yaml
+                config, errors = self.config_service.load_configuration(company_short_name)
+
+                '''
+                if errors:
+                    raise IAToolkitException(
+                        IAToolkitException.ErrorType.CONFIG_ERROR,
+                        'company.yaml validation errors'
+                    )
+                '''
+
+                # complement the instance self data
+                company_instance.company_short_name = company_short_name
+                company_instance.company = config.get('company')
+
+            except Exception as e:
+                logging.error(f"❌ Failed to register configuration for '{company_short_name}': {e}")
+                raise e
+
+        return True
+
+    def setup_iatoolkit_system(self):
+        try:
+            # system tools registration
+            self.tool_service.register_system_tools()
+
+            # system prompts registration
+            self.prompt_service.register_system_prompts()
+
+        except Exception as e:
+            self.llmquery_repo.rollback()
+            raise IAToolkitException(IAToolkitException.ErrorType.DATABASE_ERROR, str(e))
+
+
+    def dispatch(self, company_short_name: str, function_name: str, **kwargs) -> dict:
+        company_key = company_short_name.lower()
+
+        if company_key not in self.company_instances:
+            available_companies = list(self.company_instances.keys())
+            raise IAToolkitException(
+                IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
+                f"Company '{company_short_name}' not configured. available companies: {available_companies}"
+            )
+
+        # check if action is a system function using ToolService
+        if self.tool_service.is_system_tool(function_name):
+            # this is the system function to be executed.
+            handler = self.tool_service.get_system_handler(function_name)
+            logging.info(
+                f"Calling system handler [{function_name}] "
+                f"with company_short_name={company_short_name} "
+                f"and kwargs={kwargs}"
+            )
+            return handler(company_short_name, **kwargs)
+
+        company_instance = self.company_instances[company_short_name]
+        try:
+            return company_instance.handle_request(function_name, **kwargs)
+        except IAToolkitException as e:
+            # Si ya es una IAToolkitException, la relanzamos para preservar el tipo de error original.
+            raise e
+
+        except Exception as e:
+            logging.exception(e)
+            raise IAToolkitException(IAToolkitException.ErrorType.EXTERNAL_SOURCE_ERROR,
+                               f"Error in function call '{function_name}': {str(e)}") from e
+
+
+    def get_company_instance(self, company_name: str):
+        """Returns the instance for a given company name."""
+        return self.company_instances.get(company_name)

{services → iatoolkit/services}/document_service.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2024 Fernando Libedinsky
-# Producto: IAToolkit
-# Todos los derechos reservados.
-# En trámite de registro en el Registro de Propiedad Intelectual de Chile.
+# Product: IAToolkit
+#
+# IAToolkit is open source software.
 
 from docx import Document
 import fitz  # PyMuPDF
@@ -10,26 +10,34 @@ import io
 import os
 import pytesseract
 from injector import inject
-from common.exceptions import IAToolkitException
+from iatoolkit.common.exceptions import IAToolkitException
+from iatoolkit.services.i18n_service import I18nService
+from iatoolkit.services.excel_service import ExcelService
+
 
 class DocumentService:
     @inject
-    def __init__(self):
+    def __init__(self,
+                 excel_service: ExcelService,
+                 i18n_service: I18nService):
+        self.excel_service = excel_service
+        self.i18n_service = i18n_service
+
         # max number of pages to load
-        self.max_doc_pages = int(os.getenv("MAX_DOC_PAGES", "10"))
+        self.max_doc_pages = int(os.getenv("MAX_DOC_PAGES", "200"))
 
     def file_to_txt(self, filename, file_content):
         try:
             if filename.lower().endswith('.docx'):
                 return self.read_docx(file_content)
-            elif filename.lower().endswith('.txt'):
+            elif filename.lower().endswith('.txt') or filename.lower().endswith('.md'):
                 if isinstance(file_content, bytes):
                     try:
                         # decode using UTF-8
                         file_content = file_content.decode('utf-8')
                     except UnicodeDecodeError:
                         raise IAToolkitException(IAToolkitException.ErrorType.FILE_FORMAT_ERROR,
-                                           "El archivo no es texto o la codificación no es UTF-8")
+                                           self.i18n_service.t('errors.services.no_text_file'))
 
                 return file_content
             elif filename.lower().endswith('.pdf'):
@@ -37,6 +45,10 @@ class DocumentService:
                     return self.read_scanned_pdf(file_content)
                 else:
                     return self.read_pdf(file_content)
+            elif filename.lower().endswith(('.xlsx', '.xls')):
+                return self.excel_service.read_excel(file_content)
+            elif filename.lower().endswith('.csv'):
+                return self.excel_service.read_csv(file_content)
             else:
                 raise IAToolkitException(IAToolkitException.ErrorType.FILE_FORMAT_ERROR,
                                    "Formato de archivo desconocido")

iatoolkit/services/embedding_service.py

@@ -0,0 +1,148 @@
+# iatoolkit/services/embedding_service.py
+# Copyright (c) 2024 Fernando Libedinsky
+# Product: IAToolkit
+
+import os
+import base64
+import numpy as np
+from huggingface_hub import InferenceClient
+from openai import OpenAI
+from injector import inject
+from iatoolkit.services.configuration_service import ConfigurationService
+from iatoolkit.services.i18n_service import I18nService
+from iatoolkit.repositories.profile_repo import ProfileRepo
+import logging
+
+
+# Wrapper classes to create a common interface for embedding clients
+class EmbeddingClientWrapper:
+    """Abstract base class for embedding client wrappers."""
+    def __init__(self, client, model: str, dimensions: int = 1536):
+        self.client = client
+        self.model = model
+        self.dimensions = dimensions
+
+    def get_embedding(self, text: str) -> list[float]:
+        """Generates and returns an embedding for the given text."""
+        raise NotImplementedError
+
+class HuggingFaceClientWrapper(EmbeddingClientWrapper):
+    def get_embedding(self, text: str) -> list[float]:
+        embedding = self.client.feature_extraction(text)
+        # Ensure the output is a flat list of floats
+        if isinstance(embedding, list) and len(embedding) > 0 and isinstance(embedding[0], list):
+            return embedding[0]
+        return embedding
+
+class OpenAIClientWrapper(EmbeddingClientWrapper):
+    def get_embedding(self, text: str) -> list[float]:
+        # The OpenAI API expects the input text to be clean
+        text = text.replace("\n", " ")
+        response = self.client.embeddings.create(input=[text],
+                                                 model=self.model,
+                                                 dimensions=self.dimensions)
+        return response.data[0].embedding
+
+# Factory and Service classes
+class EmbeddingClientFactory:
+    """
+    Manages the lifecycle of embedding client wrappers for different companies.
+    It ensures that only one client wrapper is created per company, and it is thread-safe.
+    """
+    @inject
+    def __init__(self, config_service: ConfigurationService):
+        self.config_service = config_service
+        self._clients = {}  # Cache for storing initialized client wrappers
+
+    def get_client(self, company_short_name: str) -> EmbeddingClientWrapper:
+        """
+        Retrieves a configured embedding client wrapper for a specific company.
+        If the client is not in the cache, it creates and stores it.
+        """
+        if company_short_name in self._clients:
+            return self._clients[company_short_name]
+
+        # Get the embedding provider and model from the company.yaml
+        embedding_config = self.config_service.get_configuration(company_short_name, 'embedding_provider')
+        if not embedding_config:
+            raise ValueError(f"Embedding provider not configured for company '{company_short_name}'.")
+
+        provider = embedding_config.get('provider')
+        if not provider:
+            raise ValueError(f"Embedding provider not configured for company '{company_short_name}'.")
+        model = embedding_config.get('model')
+        dimensions = int(embedding_config.get('dimensions', "1536"))
+
+        api_key_name = embedding_config.get('api_key_name')
+        if not api_key_name:
+            raise ValueError(f"Missiong configuration for embedding_provider:api_key_name en config.yaml.")
+
+        api_key = os.getenv(api_key_name)
+        if not api_key:
+            raise ValueError(f"Environment variable '{api_key_name}' is not set.")
+
+        # Logic to handle multiple providers
+        wrapper = None
+        if provider == 'huggingface':
+            if not model:
+                model='sentence-transformers/all-MiniLM-L6-v2'
+            client = InferenceClient(model=model, token=api_key)
+            wrapper = HuggingFaceClientWrapper(client, model, dimensions)
+        elif provider == 'openai':
+            client = OpenAI(api_key=api_key)
+            if not model:
+                model='text-embedding-ada-002'
+            wrapper = OpenAIClientWrapper(client, model, dimensions)
+        else:
+            raise NotImplementedError(f"Embedding provider '{provider}' is not implemented.")
+
+        logging.debug(f"Embedding client for '{company_short_name}' created with model: {model} via {provider}")
+        self._clients[company_short_name] = wrapper
+        return wrapper
+
+class EmbeddingService:
+    """
+    A stateless service for generating text embeddings.
+    It relies on the EmbeddingClientFactory to get the correct,
+    company-specific embedding client on demand.
+    """
+    @inject
+    def __init__(self,
+                 client_factory: EmbeddingClientFactory,
+                 profile_repo: ProfileRepo,
+                 i18n_service: I18nService):
+        self.client_factory = client_factory
+        self.i18n_service = i18n_service
+        self.profile_repo = profile_repo
+
+
+    def embed_text(self, company_short_name: str, text: str, to_base64: bool = False) -> list[float] | str:
+        """
+        Generates the embedding for a given text using the appropriate company model.
+        """
+        try:
+            company = self.profile_repo.get_company_by_short_name(company_short_name)
+            if not company:
+                raise ValueError(self.i18n_service.t('errors.company_not_found', company_short_name=company_short_name))
+
+            # 1. Get the correct client wrapper from the factory
+            client_wrapper = self.client_factory.get_client(company_short_name)
+
+            # 2. Use the wrapper's common interface to get the embedding
+            embedding = client_wrapper.get_embedding(text)
+            # 3. Process the result
+            if to_base64:
+                return base64.b64encode(np.array(embedding, dtype=np.float32).tobytes()).decode('utf-8')
+
+            return embedding
+        except Exception as e:
+            logging.error(f"Error generating embedding for text: {text[:80]}... - {e}")
+            raise
+
+    def get_model_name(self, company_short_name: str) -> str:
+        """
+        Helper method to get the model name for a specific company.
+        """
+        # Get the wrapper and return the model name from it
+        client_wrapper = self.client_factory.get_client(company_short_name)
+        return client_wrapper.model

iatoolkit/services/excel_service.py

@@ -0,0 +1,156 @@
+# Copyright (c) 2024 Fernando Libedinsky
+# Product: IAToolkit
+#
+# IAToolkit is open source software.
+
+from flask import current_app, jsonify
+from iatoolkit.common.util import Utility
+import pandas as pd
+from uuid import uuid4
+from pathlib import Path
+from iatoolkit.common.exceptions import IAToolkitException
+from iatoolkit.services.i18n_service import I18nService
+from injector import inject
+import os
+import io
+import logging
+import json
+
+EXCEL_MIME = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+
+
+class ExcelService:
+    @inject
+    def __init__(self,
+                 util: Utility,
+                 i18n_service: I18nService):
+        self.util = util
+        self.i18n_service = i18n_service
+
+    def read_excel(self, file_content: bytes) -> str:
+        """
+        Reads an Excel file and converts its content to a JSON string.
+        - If the Excel file has a single sheet, it returns the JSON of that sheet.
+        - If it has multiple sheets, it returns a JSON object with sheet names as keys.
+        """
+        try:
+            # Use a BytesIO object to allow pandas to read the in-memory byte content
+            file_like_object = io.BytesIO(file_content)
+
+            # Read all sheets into a dictionary of DataFrames
+            xls = pd.read_excel(file_like_object, sheet_name=None)
+
+            if len(xls) == 1:
+                # If only one sheet, return its JSON representation directly
+                sheet_name = list(xls.keys())[0]
+                return xls[sheet_name].to_json(orient='records', indent=4)
+            else:
+                # If multiple sheets, create a dictionary of JSON strings
+                sheets_json = {}
+                for sheet_name, df in xls.items():
+                    sheets_json[sheet_name] = df.to_json(orient='records', indent=4)
+                return json.dumps(sheets_json, indent=4)
+
+        except Exception as e:
+            raise IAToolkitException(IAToolkitException.ErrorType.FILE_FORMAT_ERROR,
+                                     self.i18n_service.t('errors.services.cannot_read_excel')) from e
+
+    def read_csv(self, file_content: bytes) -> str:
+        """
+        Reads a CSV file and converts its content to a JSON string.
+        """
+        try:
+            # Use a BytesIO object to allow pandas to read the in-memory byte content
+            file_like_object = io.BytesIO(file_content)
+
+            # Read the CSV into a DataFrame
+            df = pd.read_csv(file_like_object)
+
+            # Return JSON representation
+            return df.to_json(orient='records', indent=4)
+
+        except Exception as e:
+            raise IAToolkitException(IAToolkitException.ErrorType.FILE_FORMAT_ERROR,
+                                     self.i18n_service.t('errors.services.cannot_read_csv')) from e
+
+    def excel_generator(self, company_short_name: str, **kwargs) -> str:
+        """
+        Genera un Excel a partir de una lista de diccionarios.
+
+        Parámetros esperados en kwargs:
+          - filename: str (nombre lógico a mostrar, ej. "reporte_clientes.xlsx") [obligatorio]
+          - data: list[dict] (filas del excel) [obligatorio]
+          - sheet_name: str = "hoja 1"
+
+        Retorna:
+             {
+                "filename": "reporte.xlsx",
+                "attachment_token": "8b7f8a66-...-c1c3.xlsx",
+                "content_type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                "download_link": "/download/8b7f8a66-...-c1c3.xlsx"
+                }
+        """
+        try:
+            # get the parameters
+            fname = kwargs.get('filename')
+            if not fname:
+                return self.i18n_service.t('errors.services.no_output_file')
+
+            data = kwargs.get('data')
+            if not data or not isinstance(data, list):
+                return self.i18n_service.t('errors.services.no_data_for_excel')
+
+            sheet_name = kwargs.get('sheet_name', 'hoja 1')
+
+            # 1. convert dictionary to dataframe
+            df = pd.DataFrame(data)
+
+            # 3. create temporary name
+            token = f"{uuid4()}.xlsx"
+
+            # 4. check that download directory is configured
+            if 'IATOOLKIT_DOWNLOAD_DIR' not in current_app.config:
+                return self.i18n_service.t('errors.services.no_download_directory')
+
+            download_dir = current_app.config['IATOOLKIT_DOWNLOAD_DIR']
+            filepath = Path(download_dir) / token
+            filepath.parent.mkdir(parents=True, exist_ok=True)
+
+            # 4. save excel file in temporary directory
+            df.to_excel(filepath, index=False, sheet_name=sheet_name)
+
+            # 5. return the  link to the LLM
+            return {
+                "filename": fname,
+                "attachment_token": token,
+                "content_type": EXCEL_MIME,
+                "download_link": f"/download/{token}"
+                }
+
+        except Exception as e:
+            raise IAToolkitException(IAToolkitException.ErrorType.CALL_ERROR,
+                               self.i18n_service.t('errors.services.cannot_create_excel')) from e
+
+    def validate_file_access(self, filename):
+        try:
+            if not filename:
+                return jsonify({"error": self.i18n_service.t('errors.services.invalid_filename')})
+            # Prevent path traversal attacks
+            if '..' in filename or filename.startswith('/') or '\\' in filename:
+                return jsonify({"error": self.i18n_service.t('errors.services.invalid_filename')})
+
+            temp_dir = os.path.join(current_app.root_path, 'static', 'temp')
+            file_path = os.path.join(temp_dir, filename)
+
+            if not os.path.exists(file_path):
+                return jsonify({"error": self.i18n_service.t('errors.services.file_not_exist')})
+
+            if not os.path.isfile(file_path):
+                return jsonify({"error": self.i18n_service.t('errors.services.path_is_not_a_file')})
+
+            return None
+
+        except Exception as e:
+            error_msg = f"File validation error {filename}: {str(e)}"
+            logging.error(error_msg)
+            return jsonify({"error": self.i18n_service.t('errors.services.file_validation_error')})

{services → iatoolkit/services}/file_processor_service.py

@@ -1,58 +1,70 @@
 # Copyright (c) 2024 Fernando Libedinsky
-# Producto: IAToolkit
-# Todos los derechos reservados.
-# En trámite de registro en el Registro de Propiedad Intelectual de Chile.
+# Product: IAToolkit
+#
+# IAToolkit is open source software.
 
-from infra.connectors.file_connector import FileConnector
+from iatoolkit.infra.connectors.file_connector import FileConnector
 import logging
 import os
 from typing import Optional, Callable, Dict
+from iatoolkit.repositories.models import Company
 
 
 class FileProcessorConfig:
+    """Configuration class for the FileProcessor."""
     def __init__(
         self,
         filters: Dict,
-        action: Callable[[str, bytes], None],
+        callback: Callable[[Company, str, bytes, dict], None],
         continue_on_error: bool = True,
         log_file: str = 'file_processor.log',
         echo: bool = False,
-        context: dict = None,
+        context: dict = None
     ):
+        """
+        Initializes the FileProcessor configuration.
+
+        Args:
+            filters (Dict): A dictionary of filters to apply to file names.
+                Example: {'filename_contains': '.pdf'}
+            action (Callable): The function to execute for each processed file.
+                It receives filename (str), content (bytes), and context (dict).
+            continue_on_error (bool): If True, continues processing other files upon an error.
+            log_file (str): The path to the log file.
+            echo (bool): If True, prints progress to the console.
+            context (dict): A context dictionary passed to the action function.
+        """
         self.filters = filters
-        self.action = action
+        self.callback = callback
         self.continue_on_error = continue_on_error
         self.log_file = log_file
         self.echo = echo
         self.context = context or {}
 
 class FileProcessor:
+    """
+    A generic service to process files from a given data source (connector).
+    It lists files, applies filters, and executes a specific action for each one.
+    """
     def __init__(self,
                  connector: FileConnector,
                  config: FileProcessorConfig,
                  logger: Optional[logging.Logger] = None):
         self.connector = connector
         self.config = config
-        self.logger = logger or self._setup_logger()
         self.processed_files = 0
 
-    def _setup_logger(self):
-        logging.basicConfig(
-            filename=self.config.log_file,
-            level=logging.INFO,
-            format='%(asctime)s - %(levelname)s - %(message)s'
-        )
-        return logging.getLogger(__name__)
 
     def process_files(self):
+        # Fetches files from the connector, filters them, and processes them.
         try:
             files = self.connector.list_files()
         except Exception as e:
-            self.logger.error(f"Error fetching files: {e}")
+            logging.error(f"Error fetching files: {e}")
             return False
 
         if self.config.echo:
-            print(f'cargando un total de {len(files)} archivos')
+            print(f'loading {len(files)} files')
 
         for file_info in files:
             file_path = file_info['path']
@@ -67,15 +79,18 @@ class FileProcessor:
 
                 content = self.connector.get_file_content(file_path)
 
-                # execute the action defined
+                # execute the callback function
                 filename = os.path.basename(file_name)
-                self.config.action(filename, content, self.config.context)
+                self.config.callback(company=self.config.context.get('company'),
+                                     filename=filename,
+                                     content=content,
+                                     context=self.config.context)
                 self.processed_files += 1
 
-                self.logger.info(f"Successfully processed file: {file_path}")
+                logging.info(f"Successfully processed file: {file_path}")
 
             except Exception as e:
-                self.logger.error(f"Error processing {file_path}: {e}")
+                logging.error(f"Error processing {file_path}: {e}")
                 if not self.config.continue_on_error:
                     raise e