iatoolkit 0.66.2__py3-none-any.whl → 0.71.4__py3-none-any.whl

iatoolkit/services/language_service.py

@@ -1,12 +1,13 @@
 # iatoolkit/services/language_service.py
 
 import logging
-from injector import inject
+from injector import inject, singleton
 from flask import g, request
 from iatoolkit.repositories.profile_repo import ProfileRepo
+from iatoolkit.services.configuration_service import ConfigurationService
 from iatoolkit.common.session_manager import SessionManager
 
-
+@singleton
 class LanguageService:
     """
     Determines the correct language for the current request
@@ -14,8 +15,13 @@ class LanguageService:
     and caches it in the Flask 'g' object for the request's lifecycle.
     """
 
+    FALLBACK_LANGUAGE = 'es'
+
     @inject
-    def __init__(self, profile_repo: ProfileRepo):
+    def __init__(self,
+                 config_service: ConfigurationService,
+                 profile_repo: ProfileRepo):
+        self.config_service = config_service
         self.profile_repo = profile_repo
 
     def _get_company_short_name(self) -> str | None:
@@ -49,29 +55,29 @@ class LanguageService:
         if 'lang' in g:
             return g.lang
 
-        from iatoolkit.services.i18n_service import I18nService
-        lang = I18nService.FALLBACK_LANGUAGE
-
         try:
+            # Priority 1: User's preferred language
+            user_identifier = SessionManager.get('user_identifier')
+            if user_identifier:
+                user = self.profile_repo.get_user_by_email(user_identifier)
+                if user and user.preferred_language:
+                    logging.debug(f"Language determined by user preference: {user.preferred_language}")
+                    g.lang = user.preferred_language
+                    return g.lang
+
+            # Priority 2: Company's default language
             company_short_name = self._get_company_short_name()
             if company_short_name:
-                # Prioridad 1: Preferencia del Usuario
-                user_identifier = SessionManager.get('user_identifier')
-                if user_identifier:
-                    # Usamos el repositorio para obtener el objeto User
-                    user = self.profile_repo.get_user_by_email(
-                        user_identifier)  # Asumiendo que el email es el identificador
-                    if user and user.preferred_language:
-                        g.lang = user.preferred_language
-                        return g.lang
-
-                # Prioridad 2: Idioma por defecto de la Compañía (si no se encontró preferencia de usuario)
-                company = self.profile_repo.get_company_by_short_name(company_short_name)
-                if company and company.default_language:
-                    lang = company.default_language
+                locale = self.config_service.get_configuration(company_short_name, 'locale')
+                if locale:
+                    company_language = locale.split('_')[0]
+                    g.lang = company_language
+                    return g.lang
         except Exception as e:
-            logging.debug(f"Could not determine language, falling back to default. Reason: {e}")
+            logging.info(f"Could not determine language, falling back to default. Reason: {e}")
             pass
 
-        g.lang = lang
-        return lang
+        # Priority 3: System-wide fallback
+        logging.info(f"Language determined by system fallback: {self.FALLBACK_LANGUAGE}")
+        g.lang = self.FALLBACK_LANGUAGE
+        return g.lang

iatoolkit/services/load_documents_service.py

@@ -1,50 +1,41 @@
 # Copyright (c) 2024 Fernando Libedinsky
 # Product: IAToolkit
-#
-# IAToolkit is open source software.
 
 from iatoolkit.repositories.vs_repo import VSRepo
 from iatoolkit.repositories.document_repo import DocumentRepo
-from iatoolkit.repositories.profile_repo import ProfileRepo
-from iatoolkit.repositories.llm_query_repo import LLMQueryRepo
-
 from iatoolkit.repositories.models import Document, VSDoc, Company
 from iatoolkit.services.document_service import DocumentService
+from iatoolkit.services.configuration_service import ConfigurationService
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from iatoolkit.infra.connectors.file_connector_factory import FileConnectorFactory
 from iatoolkit.services.file_processor_service import FileProcessorConfig, FileProcessor
-from iatoolkit.services.dispatcher_service import Dispatcher
 from iatoolkit.common.exceptions import IAToolkitException
 import logging
 import base64
-from injector import inject
-from typing import Dict
+from injector import inject, singleton
+import os
 
 
+@singleton
 class LoadDocumentsService:
     """
     Orchestrates the process of loading, processing, and storing documents
-    from various sources for different companies.
+    from various sources defined in the company's configuration.
     """
     @inject
     def __init__(self,
+                 config_service: ConfigurationService,
                  file_connector_factory: FileConnectorFactory,
                  doc_service: DocumentService,
                  doc_repo: DocumentRepo,
                  vector_store: VSRepo,
-                 profile_repo: ProfileRepo,
-                 dispatcher: Dispatcher,
-                 llm_query_repo: LLMQueryRepo
                  ):
+        self.config_service = config_service
         self.doc_service = doc_service
         self.doc_repo = doc_repo
-        self.profile_repo = profile_repo
-        self.llm_query_repo = llm_query_repo
         self.vector_store = vector_store
         self.file_connector_factory = file_connector_factory
-        self.dispatcher = dispatcher
 
-        # lower warnings
         logging.getLogger().setLevel(logging.ERROR)
 
         self.splitter = RecursiveCharacterTextSplitter(
@@ -53,135 +44,131 @@ class LoadDocumentsService:
             separators=["\n\n", "\n", "."]
         )
 
-    def load_company_files(self,
-                         company: Company,
-                         connector_config: Dict,
-                         predefined_metadata: Dict = None,
-                         filters: Dict = None):
+    def load_sources(self,
+                     company: Company,
+                     sources_to_load: list[str] = None,
+                     filters: dict = None) -> int:
         """
-        Loads all the company files from a connector
+        Loads documents from one or more configured sources for a company.
 
         Args:
             company (Company): The company to load files for.
-            connector_config (Dict): The configuration for the file connector.
-            predefined_metadata (Dict, optional): Metadata to be added to all documents from this source.
-            filters (Dict, optional): Filters to apply to the files.
+            sources_to_load (list[str], optional): A list of specific source names to load.
+                                                  If None, all configured sources will be loaded.
+            filters (dict, optional): Filters to apply when listing files (e.g., file extension).
 
         Returns:
-            int: The number of processed files.
+            int: The total number of processed files.
         """
-        if not connector_config:
-            raise IAToolkitException(IAToolkitException.ErrorType.MISSING_PARAMETER,
-                        f"Falta configurar conector")
+        knowledge_base_config = self.config_service.get_configuration(company.short_name, 'knowledge_base')
+        if not knowledge_base_config:
+            raise IAToolkitException(IAToolkitException.ErrorType.CONFIG_ERROR,
+                                     f"Missing 'knowledge_base' configuration for company '{company.short_name}'.")
+
+        if not sources_to_load:
+            raise IAToolkitException(IAToolkitException.ErrorType.PARAM_NOT_FILLED,
+                                f"Missing sources to load for company '{company.short_name}'.")
+
+        base_connector_config = self._get_base_connector_config(knowledge_base_config)
+        all_sources = knowledge_base_config.get('document_sources', {})
+
+        total_processed_files = 0
+        for source_name in sources_to_load:
+            source_config = all_sources.get(source_name)
+            if not source_config:
+                logging.warning(f"Source '{source_name}' not found in configuration for company '{company.short_name}'. Skipping.")
+                continue
+
+            try:
+                logging.info(f"Processing source '{source_name}' for company '{company.short_name}'...")
+
+                # Combine the base connector configuration with the specific path from the source.
+                full_connector_config = base_connector_config.copy()
+                full_connector_config['path'] = source_config.get('path')
+
+                # Prepare the context for the callback function.
+                context = {
+                    'company': company,
+                    'metadata': source_config.get('metadata', {})
+                }
+
+                processor_config = FileProcessorConfig(
+                    callback=self._file_processing_callback,
+                    context=context,
+                    filters=filters or {"filename_contains": ".pdf"},
+                    continue_on_error=True,
+                    echo=True
+                )
 
-        try:
-            if not filters:
-                filters = {"filename_contains": ".pdf"}
-
-            # Pasar metadata predefinida como parte del contexto al procesador
-            # para que esté disponible en la función load_file_callback
-            context = {
-                'company': company,
-                'metadata': {}
-            }
-
-            if predefined_metadata:
-                context['metadata'] = predefined_metadata
-
-            # config the processor
-            processor_config = FileProcessorConfig(
-                callback=self.load_file_callback,
-                context=context,
-                filters=filters,
-                continue_on_error=True,
-                echo=True
-            )
+                connector = self.file_connector_factory.create(full_connector_config)
+                processor = FileProcessor(connector, processor_config)
+                processor.process_files()
 
-            connector = self.file_connector_factory.create(connector_config)
-            processor = FileProcessor(connector, processor_config)
+                total_processed_files += processor.processed_files
+                logging.info(f"Finished processing source '{source_name}'. Processed {processor.processed_files} files.")
 
-            # process the files
-            processor.process_files()
+            except Exception as e:
+                logging.exception(f"Failed to process source '{source_name}' for company '{company.short_name}': {e}")
 
-            return processor.processed_files
-        except Exception as e:
-            logging.exception("Loading files error: %s", str(e))
-            return {"error": str(e)}
+        return total_processed_files
 
-    def load_file_callback(self, company: Company, filename: str, content: bytes, context: dict = {}):
-        """
-        Processes a single file: extracts text, generates metadata, and saves it
-        to the relational database and the vector store.
-        This method is intended to be used as the 'action' for FileProcessor.
+    def _get_base_connector_config(self, knowledge_base_config: dict) -> dict:
+        """Determines and returns the appropriate base connector configuration (dev vs prod)."""
+        connectors = knowledge_base_config.get('connectors', {})
+        env = os.getenv('FLASK_ENV', 'dev')
 
-        Args:
-            company (Company): The company associated with the file.
-            filename (str): The name of the file.
-            content (bytes): The binary content of the file.
-            context (dict, optional): A context dictionary, may contain predefined metadata.
-        """
+        if env == 'dev':
+            return connectors.get('development', {'type': 'local'})
+        else:
+            prod_config = connectors.get('production')
+            if not prod_config:
+                raise IAToolkitException(IAToolkitException.ErrorType.CONFIG_ERROR,
+                                         "Production connector configuration is missing.")
+            # The S3 connector itself is responsible for reading AWS environment variables.
+            # No need to pass credentials explicitly here.
+            return prod_config
 
+    def _file_processing_callback(self, company: Company, filename: str, content: bytes, context: dict = None):
+        """
+        Callback method to process a single file. It extracts text, merges metadata,
+        and saves the document to both relational and vector stores.
+        """
         if not company:
-            raise IAToolkitException(IAToolkitException.ErrorType.MISSING_PARAMETER,
-                        f"Falta configurar empresa")
+            raise IAToolkitException(IAToolkitException.ErrorType.MISSING_PARAMETER, "Missing company object in callback.")
 
-        # check if file exist in repositories
-        if self.doc_repo.get(company_id=company.id,filename=filename):
+        if self.doc_repo.get(company_id=company.id, filename=filename):
+            logging.debug(f"File '{filename}' already exists for company '{company.id}'. Skipping.")
             return
 
         try:
-            # extract text from the document
             document_content = self.doc_service.file_to_txt(filename, content)
-            content_base64 = base64.b64encode(content).decode('utf-8')
 
-            # generate metada based on the filename structure
-            dynamic_metadata = self.dispatcher.get_metadata_from_filename(company_name=company.short_name, filename=filename)
+            # Get predefined metadata from the context passed by the processor.
+            predefined_metadata = context.get('metadata', {}) if context else {}
 
-            # Obtener metadatos del contexto si existen
-            context_metadata = context.get('metadata', {}).copy() if context else {}
-
-            # Fusionar los metadatos. El orden de prioridad es:
-            # 1. dynamic_metadata (tiene mayor prioridad)
-            # 2. context_metadata (del parámetro context)
-            # Los valores en dynamic_metadata tendrán precedencia sobre los de context_metadata
-            final_meta = {**context_metadata, **dynamic_metadata}
-
-            # save the file in the document repositories
+            # Save the document to the relational database.
+            session = self.doc_repo.session
             new_document = Document(
                 company_id=company.id,
                 filename=filename,
                 content=document_content,
-                content_b64=content_base64,
-                meta=final_meta
+                content_b64=base64.b64encode(content).decode('utf-8'),
+                meta=predefined_metadata
             )
-
-            # insert the document into the Database (without commit)
-            session = self.doc_repo.session
             session.add(new_document)
-            session.flush()     # get the ID without commit
-
-            # split the content, and create the chunk list
-            splitted_content = self.splitter.split_text(document_content)
-            chunk_list = [
-                VSDoc(
-                    company_id=company.id,
-                    document_id=new_document.id,
-                    text=text
-                )
-                for text in splitted_content
-            ]
+            session.flush()  # Flush to get the new_document.id without committing.
 
-            # save to vector store
-            self.vector_store.add_document(chunk_list)
+            # Split into chunks and prepare for vector store.
+            chunks = self.splitter.split_text(document_content)
+            vs_docs = [VSDoc(company_id=company.id, document_id=new_document.id, text=text) for text in chunks]
 
-            # confirm the transaction
-            session.commit()
+            # Add document chunks to the vector store.
+            self.vector_store.add_document(company.short_name, vs_docs)
 
+            session.commit()
             return new_document
         except Exception as e:
             self.doc_repo.session.rollback()
-
-            # if something fails, throw exception
-            logging.exception("Error procesando el archivo %s: %s", filename, str(e))
+            logging.exception(f"Error processing file '{filename}': {e}")
             raise IAToolkitException(IAToolkitException.ErrorType.LOAD_DOCUMENT_ERROR,
-                               f"Error al procesar el archivo {filename}")
+                                     f"Error while processing file: {filename}")

iatoolkit/services/mail_service.py

@@ -4,6 +4,7 @@
 # IAToolkit is open source software.
 
 from iatoolkit.infra.mail_app import MailApp
+from iatoolkit.services.i18n_service import I18nService
 from injector import inject
 from pathlib import Path
 from iatoolkit.common.exceptions import IAToolkitException
@@ -13,18 +14,22 @@ TEMP_DIR = Path("static/temp")
 
 class MailService:
     @inject
-    def __init__(self, mail_app: MailApp):
+    def __init__(self,
+                 mail_app: MailApp,
+                 i18n_service: I18nService):
         self.mail_app = mail_app
+        self.i18n_service = i18n_service
+
 
     def _read_token_bytes(self, token: str) -> bytes:
         # Defensa simple contra path traversal
         if not token or "/" in token or "\\" in token or token.startswith("."):
             raise IAToolkitException(IAToolkitException.ErrorType.MAIL_ERROR,
-                               "attachment_token inválido")
+                               "attachment_token invalid")
         path = TEMP_DIR / token
         if not path.is_file():
             raise IAToolkitException(IAToolkitException.ErrorType.MAIL_ERROR,
-                               f"Adjunto no encontrado: {token}")
+                               f"attach file not found: {token}")
         return path.read_bytes()
 
     def send_mail(self, **kwargs):
@@ -59,4 +64,4 @@ class MailService:
             body=body,
             attachments=norm_attachments)
 
-        return 'mail enviado'
+        return self.i18n_service.t('services.mail_sent')

iatoolkit/services/profile_service.py

@@ -49,15 +49,15 @@ class ProfileService:
 
             company = self.profile_repo.get_company_by_short_name(company_short_name)
             if not company:
-                return {'success': False, "message": "Empresa no encontrada"}
+                return {'success': False, "message": "missing company"}
 
             # check that user belongs to company
             if company not in user.companies:
-                return {'success': False, "message": "Usuario no esta autorizado para esta empresa"}
+                return {'success': False, "message": self.i18n_service.t('errors.services.user_not_authorized')}
 
             if not user.verified:
                 return {'success': False,
-                        "message": "Tu cuenta no ha sido verificada. Por favor, revisa tu correo."}
+                        "message": self.i18n_service.t('errors.services.account_not_verified')}
 
             # 1. Build the local user profile dictionary here.
             # the user_profile variables are used on the LLM templates also (see in query_main.prompt)
@@ -74,7 +74,7 @@ class ProfileService:
 
             # 3. create the web session
             self.set_session_for_user(company.short_name, user_identifier)
-            return {'success': True, "user_identifier": user_identifier, "message": "Login exitoso"}
+            return {'success': True, "user_identifier": user_identifier, "message": "Login ok"}
         except Exception as e:
             logging.error(f"Error in login: {e}")
             return {'success': False, "message": str(e)}
@@ -95,6 +95,9 @@ class ProfileService:
             user_identifier=user_identifier,
             user_profile=external_user_profile)
 
+        # 3. make sure the flask session is clean
+        SessionManager.clear()
+
     def save_user_profile(self, company: Company, user_identifier: str, user_profile: dict):
         """
         Private helper: Takes a pre-built profile, saves it to Redis, and sets the Flask cookie.
@@ -151,7 +154,7 @@ class ProfileService:
         except Exception as e:
             # Log the error and return a generic failure message.
             logging.error(f"Failed to update language for {user_identifier}: {e}")
-            return {'success': False, 'error_message': self.i18n_service.t('errors.general.unexpected_error')}
+            return {'success': False, 'error_message': self.i18n_service.t('errors.general.unexpected_error', error=str(e))}
 
 
     def get_profile_by_identifier(self, company_short_name: str, user_identifier: str) -> dict:
@@ -230,7 +233,7 @@ class ProfileService:
 
             return {"message": self.i18n_service.t('flash_messages.signup_success')}
         except Exception as e:
-            return {"error": self.i18n_service.t('errors.general.unexpected_error')}
+            return {"error": self.i18n_service.t('errors.general.unexpected_error', error=str(e))}
 
     def update_user(self, email: str, **kwargs) -> User:
         return self.profile_repo.update_user(email, **kwargs)
@@ -324,7 +327,7 @@ class ProfileService:
     def new_api_key(self, company_short_name: str):
         company = self.get_company_by_short_name(company_short_name)
         if not company:
-            return {"error": f"la empresa {company_short_name} no existe"}
+            return {"error": self.i18n_service.t('errors.company_not_found', company_short_name=company_short_name)}
 
         length = 40     # lenght of the api key
         alphabet = string.ascii_letters + string.digits

iatoolkit/services/prompt_manager_service.py

@@ -5,21 +5,25 @@
 
 from injector import inject
 from iatoolkit.repositories.llm_query_repo import LLMQueryRepo
-
-import logging
+from iatoolkit.services.i18n_service import I18nService
 from iatoolkit.repositories.profile_repo import ProfileRepo
 from collections import defaultdict
 from iatoolkit.repositories.models import Prompt, PromptCategory, Company
 import os
 from iatoolkit.common.exceptions import IAToolkitException
 import importlib.resources
+import logging
 
 
 class PromptService:
     @inject
-    def __init__(self, llm_query_repo: LLMQueryRepo, profile_repo: ProfileRepo):
+    def __init__(self,
+                 llm_query_repo: LLMQueryRepo,
+                 profile_repo: ProfileRepo,
+                 i18n_service: I18nService):
         self.llm_query_repo = llm_query_repo
         self.profile_repo = profile_repo
+        self.i18n_service = i18n_service
 
     def create_prompt(self,
                       prompt_name: str,
@@ -36,20 +40,20 @@ class PromptService:
         if is_system_prompt:
             if not importlib.resources.files('iatoolkit.system_prompts').joinpath(prompt_filename).is_file():
                 raise IAToolkitException(IAToolkitException.ErrorType.INVALID_NAME,
-                                f'No existe el archivo de prompt de sistemas: {prompt_filename}')
+                                f'missing system prompt file: {prompt_filename}')
         else:
             template_dir = f'companies/{company.short_name}/prompts'
 
             relative_prompt_path = os.path.join(template_dir, prompt_filename)
             if not os.path.exists(relative_prompt_path):
                 raise IAToolkitException(IAToolkitException.ErrorType.INVALID_NAME,
-                               f'No existe el archivo de prompt: {relative_prompt_path}')
+                               f'missing prompt file: {relative_prompt_path}')
 
         if custom_fields:
             for f in custom_fields:
                 if ('data_key' not in f) or ('label' not in f):
                     raise IAToolkitException(IAToolkitException.ErrorType.INVALID_PARAMETER,
-                               f'El campo custom_fields debe contener los campos: data_key y label')
+                               f'The field "custom_fields" must contain the following keys: data_key y label')
 
                 # add default value for data_type
                 if 'type' not in f:
@@ -82,20 +86,20 @@ class PromptService:
             user_prompt = self.llm_query_repo.get_prompt_by_name(company, prompt_name)
             if not user_prompt:
                 raise IAToolkitException(IAToolkitException.ErrorType.DOCUMENT_NOT_FOUND,
-                                   f"No se encontró el prompt '{prompt_name}' para la empresa '{company.short_name}'")
+                                   f"prompt not found '{prompt_name}' for company '{company.short_name}'")
 
             prompt_file = f'companies/{company.short_name}/prompts/{user_prompt.filename}'
             absolute_filepath = os.path.join(execution_dir, prompt_file)
             if not os.path.exists(absolute_filepath):
                 raise IAToolkitException(IAToolkitException.ErrorType.FILE_IO_ERROR,
-                                   f"El archivo para el prompt '{prompt_name}' no existe: {absolute_filepath}")
+                                   f"prompt file '{prompt_name}' does not exist: {absolute_filepath}")
 
             try:
                 with open(absolute_filepath, 'r', encoding='utf-8') as f:
                     user_prompt_content = f.read()
             except Exception as e:
                 raise IAToolkitException(IAToolkitException.ErrorType.FILE_IO_ERROR,
-                                   f"Error leyendo el archivo de prompt '{prompt_name}' en {absolute_filepath}: {e}")
+                                   f"error while reading prompt: '{prompt_name}' in this pathname {absolute_filepath}: {e}")
 
             return user_prompt_content
 
@@ -105,9 +109,9 @@ class PromptService:
             raise
         except Exception as e:
             logging.exception(
-                f"Error al obtener el contenido del prompt para la empresa '{company.short_name}' y prompt '{prompt_name}': {e}")
+                f"error loading prompt '{prompt_name}' content for '{company.short_name}': {e}")
             raise IAToolkitException(IAToolkitException.ErrorType.PROMPT_ERROR,
-                               f'Error al obtener el contenido del prompt "{prompt_name}" para la empresa {company.short_name}: {str(e)}')
+                               f'error loading prompt "{prompt_name}" content for company {company.short_name}: {str(e)}')
 
     def get_system_prompt(self):
         try:
@@ -121,10 +125,10 @@ class PromptService:
                     content = importlib.resources.read_text('iatoolkit.system_prompts', prompt.filename)
                     system_prompt_content.append(content)
                 except FileNotFoundError:
-                    logging.warning(f"El archivo para el prompt de sistema no existe en el paquete: {prompt.filename}")
+                    logging.warning(f"Prompt file does not exist in the package: {prompt.filename}")
                 except Exception as e:
                     raise IAToolkitException(IAToolkitException.ErrorType.FILE_IO_ERROR,
-                                             f"Error leyendo el archivo de prompt del sistema '{prompt.filename}': {e}")
+                                             f"error reading system prompt '{prompt.filename}': {e}")
 
             # join the system prompts into a single string
             return "\n".join(system_prompt_content)
@@ -135,14 +139,14 @@ class PromptService:
             logging.exception(
                 f"Error al obtener el contenido del prompt de sistema: {e}")
             raise IAToolkitException(IAToolkitException.ErrorType.PROMPT_ERROR,
-                               f'Error al obtener el contenido de los prompts de sistema": {str(e)}')
+                               f'error reading the system prompts": {str(e)}')
 
     def get_user_prompts(self, company_short_name: str) -> dict:
         try:
             # validate company
             company = self.profile_repo.get_company_by_short_name(company_short_name)
             if not company:
-                return {'error': f'No existe la empresa: {company_short_name}'}
+                return {"error": self.i18n_service.t('errors.company_not_found', company_short_name=company_short_name)}
 
             # get all the prompts
             all_prompts = self.llm_query_repo.get_prompts(company)
@@ -183,6 +187,6 @@ class PromptService:
             return {'message': categorized_prompts}
 
         except Exception as e:
-            logging.error(f"Error en get_prompts: {e}")
+            logging.error(f"error in get_prompts: {e}")
             return {'error': str(e)}