iatoolkit 0.91.1__py3-none-any.whl → 1.4.2__py3-none-any.whl

iatoolkit/services/configuration_service.py

@@ -2,11 +2,10 @@
 # Copyright (c) 2024 Fernando Libedinsky
 # Product: IAToolkit
 
-from pathlib import Path
 from iatoolkit.repositories.models import Company
+from iatoolkit.common.interfaces.asset_storage import AssetRepository, AssetType
 from iatoolkit.repositories.llm_query_repo import LLMQueryRepo
 from iatoolkit.repositories.profile_repo import ProfileRepo
-from iatoolkit.common.exceptions import IAToolkitException
 from iatoolkit.common.util import Utility
 from injector import inject
 import logging
@@ -21,9 +20,11 @@ class ConfigurationService:
 
     @inject
     def __init__(self,
+                 asset_repo: AssetRepository,
                  llm_query_repo: LLMQueryRepo,
                  profile_repo: ProfileRepo,
                  utility: Utility):
+        self.asset_repo = asset_repo
         self.llm_query_repo = llm_query_repo
         self.profile_repo = profile_repo
         self.utility = utility
@@ -45,7 +46,30 @@ class ConfigurationService:
         self._ensure_config_loaded(company_short_name)
         return self._loaded_configs[company_short_name].get(content_key)
 
-    def load_configuration(self, company_short_name: str, company_instance):
+    def get_llm_configuration(self, company_short_name: str):
+        """
+        Convenience helper to obtain the 'llm' configuration block for a company.
+        Kept separate from get_configuration() to avoid coupling tests that
+        assert the number of calls to get_configuration().
+        """
+        default_llm_model = None
+        available_llm_models = []
+        self._ensure_config_loaded(company_short_name)
+        llm_config = self._loaded_configs[company_short_name].get("llm")
+        if llm_config:
+            default_llm_model = llm_config.get("model")
+            available_llm_models = llm_config.get('available_models') or []
+
+        # fallback: if no explicit list of models is provided, use the default model
+        if not available_llm_models and default_llm_model:
+            available_llm_models = [{
+                "id": default_llm_model,
+                "label": default_llm_model,
+                "description": "Modelo por defecto configurado para esta compañía."
+            }]
+        return default_llm_model, available_llm_models
+
+    def load_configuration(self, company_short_name: str):
         """
         Main entry point for configuring a company instance.
         This method is invoked by the dispatcher for each registered company.
@@ -54,71 +78,99 @@ class ConfigurationService:
 
         # 1. Load the main configuration file and supplementary content files
         config = self._load_and_merge_configs(company_short_name)
+        if config:
+            # 2. create/update company in database
+            self._register_company_database(config)
 
-        # 2. Register core company details and get the database object
-        self._register_core_details(company_instance, config)
-
-        # 3. Register databases
-        self._register_data_sources(company_short_name, config)
+            # 3. Register tools
+            self._register_tools(company_short_name, config)
 
-        # 4. Register tools
-        self._register_tools(company_instance, config)
+            # 4. Register prompt categories and prompts
+            self._register_prompts(company_short_name, config)
 
-        # 5. Register prompt categories and prompts
-        self._register_prompts(company_instance, config)
-
-        # 6. Link the persisted Company object back to the running instance
-        company_instance.company_short_name = company_short_name
-        company_instance.id = company_instance.company.id
+            # 5. Register Knowledge base information
+            self._register_knowledge_base(company_short_name, config)
 
         # Final step: validate the configuration against platform
-        self._validate_configuration(company_short_name, config)
+        errors = self._validate_configuration(company_short_name, config)
 
         logging.info(f"✅ Company '{company_short_name}' configured successfully.")
-
+        return config, errors
 
     def _load_and_merge_configs(self, company_short_name: str) -> dict:
         """
         Loads the main company.yaml and merges data from supplementary files
-        specified in the 'content_files' section.
+        specified in the 'content_files' section using AssetRepository.
         """
-        config_dir = Path("companies") / company_short_name / "config"
-        main_config_path = config_dir / "company.yaml"
-
-        if not main_config_path.exists():
-            raise FileNotFoundError(f"Main configuration file not found: {main_config_path}")
-
-        config = self.utility.load_schema_from_yaml(main_config_path)
+        main_config_filename = "company.yaml"
+
+        # verify existence of the main configuration file
+        if not self.asset_repo.exists(company_short_name, AssetType.CONFIG, main_config_filename):
+            # raise FileNotFoundError(f"Main configuration file not found: {main_config_filename}")
+            logging.exception(f"Main configuration file not found: {main_config_filename}")
+
+            # return the minimal configuration needed for starting the IAToolkit
+            # this is a for solving a chicken/egg problem when trying to migrate the configuration
+            # from filesystem to database in enterprise installation
+            # see create_assets cli command in enterprise-iatoolkit)
+            return {
+                'id': company_short_name,
+                'name': company_short_name,
+                'llm': {'model': 'gpt-5', 'provider_api_keys': {'openai':''} },
+                }
+
+        # read text and parse
+        yaml_content = self.asset_repo.read_text(company_short_name, AssetType.CONFIG, main_config_filename)
+        config = self.utility.load_yaml_from_string(yaml_content)
+        if not config:
+            return {}
 
         # Load and merge supplementary content files (e.g., onboarding_cards)
-        for key, file_path in config.get('help_files', {}).items():
-            supplementary_path = config_dir / file_path
-            if supplementary_path.exists():
-                config[key] = self.utility.load_schema_from_yaml(supplementary_path)
+        for key, filename in config.get('help_files', {}).items():
+            if self.asset_repo.exists(company_short_name, AssetType.CONFIG, filename):
+                supp_content = self.asset_repo.read_text(company_short_name, AssetType.CONFIG, filename)
+                config[key] = self.utility.load_yaml_from_string(supp_content)
             else:
-                logging.warning(f"⚠️  Warning: Content file not found: {supplementary_path}")
-                config[key] = None  # Ensure the key exists but is empty
+                logging.warning(f"⚠️  Warning: Content file not found: {filename}")
+                config[key] = None
 
         return config
 
-    def _register_core_details(self, company_instance, config: dict) -> Company:
+    def _register_company_database(self, config: dict) -> Company:
         # register the company in the database: create_or_update logic
+        if not config:
+            return None
 
-        company_obj = Company(short_name=config['id'],
-                              name=config['name'],
+        # create or update the company in database
+        company_obj = Company(short_name=config.get('id'),
+                              name=config.get('name'),
                               parameters=config.get('parameters', {}))
         company = self.profile_repo.create_company(company_obj)
 
-        # save company object with the instance
-        company_instance.company = company
+        # save company object with the configuration
+        config['company'] = company
+
         return company
 
-    def _register_data_sources(self, company_short_name: str, config: dict):
+    def register_data_sources(self,
+                              company_short_name: str,
+                              config: dict = None):
         """
         Reads the data_sources config and registers databases with SqlService.
         Uses Lazy Loading to avoid circular dependency.
+
+        Public method: Can be called externally after initialization (e.g. by Enterprise)
+        to re-register sources once new factories (like 'bridge') are available.
         """
-        # Lazy import to avoid circular dependency: ConfigService -> SqlService -> I18n -> ConfigService
+
+        # If config is not provided, try to load it from cache
+        if config is None:
+            self._ensure_config_loaded(company_short_name)
+            config = self._loaded_configs.get(company_short_name)
+
+        if not config:
+            return
+
         from iatoolkit import current_iatoolkit
         from iatoolkit.services.sql_service import SqlService
         sql_service = current_iatoolkit().get_injector().get(SqlService)
@@ -129,25 +181,46 @@ class ConfigurationService:
         if not sql_sources:
             return
 
-        logging.info(f"🛢️ Registering databases for '{company_short_name}'...")
+        logging.info(f"🛢️ Registering databases  for '{company_short_name}'...")
 
-        for db_config in sql_sources:
-            db_name = db_config.get('database')
-            db_schema = db_config.get('schema', 'public')
-            db_env_var = db_config.get('connection_string_env')
-
-            # resolve the URI
-            db_uri = os.getenv(db_env_var) if db_env_var else None
+        for source in sql_sources:
+            db_name = source.get('database')
+            if not db_name:
+                continue
 
-            if not db_uri:
+            # Prepare the config dictionary for the factory
+            db_config = {
+                'database': db_name,
+                'schema': source.get('schema', 'public'),
+                'connection_type': source.get('connection_type', 'direct'),
+
+                # Pass through keys needed for Bridge or other plugins
+                'bridge_id': source.get('bridge_id'),
+                'timeout': source.get('timeout')
+            }
+
+            # Resolve URI if env var is present (Required for 'direct', optional for others)
+            db_env_var = source.get('connection_string_env')
+            if db_env_var:
+                db_uri = os.getenv(db_env_var)
+                if db_uri:
+                    db_config['db_uri'] = db_uri
+
+            # Validation: 'direct' connections MUST have a URI
+            if db_config['connection_type'] == 'direct' and not db_config.get('db_uri'):
                 logging.error(
                     f"-> Skipping DB '{db_name}' for '{company_short_name}': missing URI in env '{db_env_var}'.")
                 continue
 
+            elif db_config['connection_type'] == 'bridge' and not db_config.get('bridge_id'):
+                logging.error(
+                    f"-> Skipping DB '{db_name}' for '{company_short_name}': missing bridge_id in configuration.")
+                continue
+
             # Register with the SQL service
-            sql_service.register_database(db_uri, db_name, db_schema)
+            sql_service.register_database(company_short_name, db_name, db_config)
 
-    def _register_tools(self, company_instance, config: dict):
+    def _register_tools(self, company_short_name: str, config: dict):
         """creates in the database each tool defined in the YAML."""
         # Lazy import and resolve ToolService locally
         from iatoolkit import current_iatoolkit
@@ -155,9 +228,9 @@ class ConfigurationService:
         tool_service = current_iatoolkit().get_injector().get(ToolService)
 
         tools_config = config.get('tools', [])
-        tool_service.sync_company_tools(company_instance, tools_config)
+        tool_service.sync_company_tools(company_short_name, tools_config)
 
-    def _register_prompts(self, company_instance, config: dict):
+    def _register_prompts(self, company_short_name: str, config: dict):
         """
          Delegates prompt synchronization to PromptService.
          """
@@ -170,11 +243,24 @@ class ConfigurationService:
         categories_config = config.get('prompt_categories', [])
 
         prompt_service.sync_company_prompts(
-            company_instance=company_instance,
+            company_short_name=company_short_name,
             prompts_config=prompts_config,
             categories_config=categories_config
         )
 
+    def _register_knowledge_base(self, company_short_name: str, config: dict):
+        # Lazy import to avoid circular dependency
+        from iatoolkit import current_iatoolkit
+        from iatoolkit.services.knowledge_base_service import KnowledgeBaseService
+        knowledge_base = current_iatoolkit().get_injector().get(KnowledgeBaseService)
+
+        kb_config = config.get('knowledge_base', {})
+        categories_config = kb_config.get('collections', [])
+
+        # sync collection types in database
+        knowledge_base.sync_collection_types(company_short_name, categories_config)
+
+
     def _validate_configuration(self, company_short_name: str, config: dict):
         """
         Validates the structure and consistency of the company.yaml configuration.
@@ -182,13 +268,15 @@ class ConfigurationService:
         Raises IAToolkitException if any validation error is found.
         """
         errors = []
-        config_dir = Path("companies") / company_short_name / "config"
-        prompts_dir = Path("companies") / company_short_name / "prompts"
 
         # Helper to collect errors
         def add_error(section, message):
             errors.append(f"[{section}] {message}")
 
+        if not config:
+            add_error("General", "Configuration file missing or with errors, check the application logs.")
+            return errors
+
         # 1. Top-level keys
         if not config.get("id"):
             add_error("General", "Missing required key: 'id'")
@@ -204,8 +292,8 @@ class ConfigurationService:
         else:
             if not config.get("llm", {}).get("model"):
                 add_error("llm", "Missing required key: 'model'")
-            if not config.get("llm", {}).get("api-key"):
-                add_error("llm", "Missing required key: 'api-key'")
+            if not config.get("llm", {}).get("provider_api_keys"):
+                add_error("llm", "Missing required key: 'provider_api_keys'")
 
         # 3. Embedding Provider
         if isinstance(config.get("embedding_provider"), dict):
@@ -220,8 +308,12 @@ class ConfigurationService:
         for i, source in enumerate(config.get("data_sources", {}).get("sql", [])):
             if not source.get("database"):
                 add_error(f"data_sources.sql[{i}]", "Missing required key: 'database'")
-            if not source.get("connection_string_env"):
+
+            connection_type = source.get("connection_type")
+            if connection_type == 'direct' and not source.get("connection_string_env"):
                 add_error(f"data_sources.sql[{i}]", "Missing required key: 'connection_string_env'")
+            elif connection_type == 'bridge' and not source.get("bridge_id"):
+                add_error(f"data_sources.sql[{i}]", "Missing bridge_id'")
 
         # 5. Tools
         for i, tool in enumerate(config.get("tools", [])):
@@ -242,9 +334,9 @@ class ConfigurationService:
             if not prompt_name:
                 add_error(f"prompts[{i}]", "Missing required key: 'name'")
             else:
-                prompt_file = prompts_dir / f"{prompt_name}.prompt"
-                if not prompt_file.is_file():
-                    add_error(f"prompts/{prompt_name}:", f"Prompt file not found: {prompt_file}")
+                prompt_filename = f"{prompt_name}.prompt"
+                if not self.asset_repo.exists(company_short_name, AssetType.PROMPT, prompt_filename):
+                    add_error(f"prompts/{prompt_name}:", f"Prompt file not found: {prompt_filename}")
 
                 prompt_description = prompt.get("description")
                 if not prompt_description:
@@ -289,9 +381,9 @@ class ConfigurationService:
             if not filename:
                 add_error(f"help_files.{key}", "Filename cannot be empty.")
                 continue
-            help_file_path = config_dir / filename
-            if not help_file_path.is_file():
-                add_error(f"help_files.{key}", f"Help file not found: {help_file_path}")
+            if not self.asset_repo.exists(company_short_name, AssetType.CONFIG, filename):
+                add_error(f"help_files.{key}", f"Help file not found: {filename}")
+
 
         # If any errors were found, log all messages and raise an exception
         if errors:
@@ -299,8 +391,5 @@ class ConfigurationService:
                 f" - {e}" for e in errors)
             logging.error(error_summary)
 
-            raise IAToolkitException(
-                IAToolkitException.ErrorType.CONFIG_ERROR,
-                'company.yaml validation errors'
-            )
+        return errors
 

iatoolkit/services/dispatcher_service.py

@@ -59,13 +59,25 @@ class Dispatcher:
         self.setup_iatoolkit_system()
 
         # Loads the configuration of every company: company.yaml file
-        for company_name, company_instance in self.company_instances.items():
+        for company_short_name, company_instance in self.company_instances.items():
             try:
                 # read company configuration from company.yaml
-                self.config_service.load_configuration(company_name, company_instance)
+                config, errors = self.config_service.load_configuration(company_short_name)
+
+                '''
+                if errors:
+                    raise IAToolkitException(
+                        IAToolkitException.ErrorType.CONFIG_ERROR,
+                        'company.yaml validation errors'
+                    )
+                '''
+
+                # complement the instance self data
+                company_instance.company_short_name = company_short_name
+                company_instance.company = config.get('company')
 
             except Exception as e:
-                logging.error(f"❌ Failed to register configuration for '{company_name}': {e}")
+                logging.error(f"❌ Failed to register configuration for '{company_short_name}': {e}")
                 raise e
 
         return True
@@ -95,7 +107,13 @@ class Dispatcher:
 
         # check if action is a system function using ToolService
         if self.tool_service.is_system_tool(function_name):
+            # this is the system function to be executed.
             handler = self.tool_service.get_system_handler(function_name)
+            logging.info(
+                f"Calling system handler [{function_name}] "
+                f"with company_short_name={company_short_name} "
+                f"and kwargs={kwargs}"
+            )
             return handler(company_short_name, **kwargs)
 
         company_instance = self.company_instances[company_short_name]

iatoolkit/services/file_processor_service.py

@@ -74,9 +74,6 @@ class FileProcessor:
                 if not self._apply_filters(file_name):
                     continue
 
-                if self.config.echo:
-                    print(f'loading: {file_name}')
-
                 content = self.connector.get_file_content(file_path)
 
                 # execute the callback function
@@ -87,8 +84,6 @@ class FileProcessor:
                                      context=self.config.context)
                 self.processed_files += 1
 
-                logging.info(f"Successfully processed file: {file_path}")
-
             except Exception as e:
                 logging.error(f"Error processing {file_path}: {e}")
                 if not self.config.continue_on_error:

iatoolkit/services/history_manager_service.py

@@ -1,9 +1,15 @@
+# Copyright (c) 2024 Fernando Libedinsky
+# Product: IAToolkit
+#
+# IAToolkit is open source software.
+
+
 import logging
 import json
-from typing import Dict, Any, Tuple, Optional
+from typing import Dict, Any, Optional
 from iatoolkit.services.user_session_context_service import UserSessionContextService
 from iatoolkit.services.i18n_service import I18nService
-from iatoolkit.infra.llm_client import llmClient
+from iatoolkit.services.llm_client_service import llmClient
 from iatoolkit.repositories.models import Company
 from iatoolkit.repositories.llm_query_repo import LLMQueryRepo
 from iatoolkit.repositories.profile_repo import ProfileRepo
@@ -19,7 +25,7 @@ class HistoryManagerService:
     3. Database persistence retrieval (full chat history).
     """
     TYPE_SERVER_SIDE = 'server_side'  # For models like OpenAI
-    TYPE_CLIENT_SIDE = 'client_side'  # For models like Gemini
+    TYPE_CLIENT_SIDE = 'client_side'  # For models like Gemini and Deepseek
 
     GEMINI_MAX_TOKENS_CONTEXT_HISTORY = 200000
 
@@ -47,7 +53,7 @@ class HistoryManagerService:
         Initializes a new conversation history.
         """
         # 1. Clear existing history
-        self.session_context.clear_llm_history(company_short_name, user_identifier)
+        self.session_context.clear_llm_history(company_short_name, user_identifier, model=model)
 
         if history_type == self.TYPE_SERVER_SIDE:
             # OpenAI: Send system prompt to API and store the resulting ID
@@ -56,14 +62,14 @@ class HistoryManagerService:
                 company_base_context=prepared_context,
                 model=model
             )
-            self.session_context.save_last_response_id(company_short_name, user_identifier, response_id)
-            self.session_context.save_initial_response_id(company_short_name, user_identifier, response_id)
+            self.session_context.save_last_response_id(company_short_name, user_identifier, response_id, model=model)
+            self.session_context.save_initial_response_id(company_short_name, user_identifier, response_id, model=model)
             return {'response_id': response_id}
 
         elif history_type == self.TYPE_CLIENT_SIDE:
             # Gemini: Store system prompt as the first message in the list
             context_history = [{"role": "user", "content": prepared_context}]
-            self.session_context.save_context_history(company_short_name, user_identifier, context_history)
+            self.session_context.save_context_history(company_short_name, user_identifier, context_history, model=model)
             return {}
 
         return {}
@@ -76,14 +82,16 @@ class HistoryManagerService:
         Populates the request_params within the HistoryHandle.
         Returns True if a rebuild is needed, False otherwise.
         """
+        model = getattr(handle, "model", None)
+
         if handle.type == self.TYPE_SERVER_SIDE:
-            previous_response_id = None
             if ignore_history:
-                previous_response_id = self.session_context.get_initial_response_id(handle.company_short_name,
-                                                                                    handle.user_identifier)
+                previous_response_id = self.session_context.get_initial_response_id(
+                    handle.company_short_name,handle.user_identifier,model=model)
             else:
-                previous_response_id = self.session_context.get_last_response_id(handle.company_short_name,
-                                                                                 handle.user_identifier)
+                previous_response_id = self.session_context.get_last_response_id(
+                    handle.company_short_name,handle.user_identifier,model=model)
+
 
             if not previous_response_id:
                 handle.request_params = {}
@@ -93,8 +101,8 @@ class HistoryManagerService:
             return False
 
         elif handle.type == self.TYPE_CLIENT_SIDE:
-            context_history = self.session_context.get_context_history(handle.company_short_name,
-                                                                       handle.user_identifier) or []
+            context_history = self.session_context.get_context_history(
+                handle.company_short_name,handle.user_identifier,model=model) or []
 
             if not context_history:
                 handle.request_params = {}
@@ -104,7 +112,7 @@ class HistoryManagerService:
                 # Keep only system prompt
                 context_history = [context_history[0]]
 
-            # For Gemini, we append the current user turn to the context sent to the API
+            # Append the current user turn to the context sent to the API
             context_history.append({"role": "user", "content": user_turn_prompt})
 
             self._trim_context_history(context_history)
@@ -125,15 +133,23 @@ class HistoryManagerService:
         history_type = history_handle.type
         company_short_name = history_handle.company_short_name
         user_identifier = history_handle.user_identifier
+        model = getattr(history_handle, "model", None)
 
         if history_type == self.TYPE_SERVER_SIDE:
             if "response_id" in response:
-                self.session_context.save_last_response_id(company_short_name, user_identifier,
-                                                               response["response_id"])
+                self.session_context.save_last_response_id(
+                    company_short_name,
+                    user_identifier,
+                    response["response_id"],
+                    model=model)
 
         elif history_type == self.TYPE_CLIENT_SIDE:
-            context_history = self.session_context.get_context_history(company_short_name,
-                                                                       user_identifier) or []
+            # get the history for this company/user/model
+            context_history = self.session_context.get_context_history(
+                company_short_name,
+                user_identifier,
+                model=model)
+
             # Ensure the user prompt is recorded if not already.
             # We check content equality to handle the case where the previous message was
             # also 'user' (e.g., System Prompt) but different content.
@@ -142,10 +158,14 @@ class HistoryManagerService:
             if last_content != user_turn_prompt:
                 context_history.append({"role": "user", "content": user_turn_prompt})
 
-            if response.get('output'):
-                context_history.append({"role": "model", "content": response['output']})
+            if response.get('answer'):
+                context_history.append({"role": "assistant", "content": response.get('answer', '')})
 
-            self.session_context.save_context_history(company_short_name, user_identifier, context_history)
+            self.session_context.save_context_history(
+                company_short_name,
+                user_identifier,
+                context_history,
+                model=model)
 
     def _trim_context_history(self, context_history: list):
         """Internal helper to keep token usage within limits for client-side history."""
@@ -169,8 +189,7 @@ class HistoryManagerService:
             except IndexError:
                 break
 
-    # --- Database History Management (Legacy HistoryService) ---
-
+    # --- this is for the history popup in the chat page
     def get_full_history(self, company_short_name: str, user_identifier: str) -> dict:
         """Retrieves the full persisted history from the database."""
         try: