iatoolkit 0.91.1__py3-none-any.whl → 1.7.0__py3-none-any.whl

iatoolkit/services/configuration_service.py

@@ -2,11 +2,10 @@
 # Copyright (c) 2024 Fernando Libedinsky
 # Product: IAToolkit
 
-from pathlib import Path
 from iatoolkit.repositories.models import Company
+from iatoolkit.common.interfaces.asset_storage import AssetRepository, AssetType
 from iatoolkit.repositories.llm_query_repo import LLMQueryRepo
 from iatoolkit.repositories.profile_repo import ProfileRepo
-from iatoolkit.common.exceptions import IAToolkitException
 from iatoolkit.common.util import Utility
 from injector import inject
 import logging
@@ -21,9 +20,11 @@ class ConfigurationService:
 
     @inject
     def __init__(self,
+                 asset_repo: AssetRepository,
                  llm_query_repo: LLMQueryRepo,
                  profile_repo: ProfileRepo,
                  utility: Utility):
+        self.asset_repo = asset_repo
         self.llm_query_repo = llm_query_repo
         self.profile_repo = profile_repo
         self.utility = utility
@@ -45,7 +46,30 @@ class ConfigurationService:
         self._ensure_config_loaded(company_short_name)
         return self._loaded_configs[company_short_name].get(content_key)
 
-    def load_configuration(self, company_short_name: str, company_instance):
+    def get_llm_configuration(self, company_short_name: str):
+        """
+        Convenience helper to obtain the 'llm' configuration block for a company.
+        Kept separate from get_configuration() to avoid coupling tests that
+        assert the number of calls to get_configuration().
+        """
+        default_llm_model = None
+        available_llm_models = []
+        self._ensure_config_loaded(company_short_name)
+        llm_config = self._loaded_configs[company_short_name].get("llm")
+        if llm_config:
+            default_llm_model = llm_config.get("model")
+            available_llm_models = llm_config.get('available_models') or []
+
+        # fallback: if no explicit list of models is provided, use the default model
+        if not available_llm_models and default_llm_model:
+            available_llm_models = [{
+                "id": default_llm_model,
+                "label": default_llm_model,
+                "description": "Modelo por defecto configurado para esta compañía."
+            }]
+        return default_llm_model, available_llm_models
+
+    def load_configuration(self, company_short_name: str):
         """
         Main entry point for configuring a company instance.
         This method is invoked by the dispatcher for each registered company.
@@ -54,71 +78,234 @@ class ConfigurationService:
 
         # 1. Load the main configuration file and supplementary content files
         config = self._load_and_merge_configs(company_short_name)
+        if config:
+            # 2. create/update company in database
+            self._register_company_database(config)
 
-        # 2. Register core company details and get the database object
-        self._register_core_details(company_instance, config)
-
-        # 3. Register databases
-        self._register_data_sources(company_short_name, config)
-
-        # 4. Register tools
-        self._register_tools(company_instance, config)
+            # 3. Register tools
+            self._register_tools(company_short_name, config)
 
-        # 5. Register prompt categories and prompts
-        self._register_prompts(company_instance, config)
+            # 4. Register prompt categories and prompts
+            self._register_prompts(company_short_name, config)
 
-        # 6. Link the persisted Company object back to the running instance
-        company_instance.company_short_name = company_short_name
-        company_instance.id = company_instance.company.id
+            # 5. Register Knowledge base information
+            self._register_knowledge_base(company_short_name, config)
 
         # Final step: validate the configuration against platform
-        self._validate_configuration(company_short_name, config)
+        errors = self._validate_configuration(company_short_name, config)
 
         logging.info(f"✅ Company '{company_short_name}' configured successfully.")
+        return config, errors
 
+    def update_configuration_key(self, company_short_name: str, key: str, value) -> tuple[dict, list[str]]:
+        """
+        Updates a specific key in the company's configuration file, validates the result,
+        and saves it to the asset repository if valid.
 
-    def _load_and_merge_configs(self, company_short_name: str) -> dict:
+        Args:
+            company_short_name: The company identifier.
+            key: The configuration key to update (supports dot notation, e.g., 'llm.model').
+            value: The new value for the key.
+
+        Returns:
+            A tuple containing the updated configuration dict and a list of error strings (if any).
         """
-        Loads the main company.yaml and merges data from supplementary files
-        specified in the 'content_files' section.
+        # 1. Load raw config from file (to avoid working with merged supplementary files if possible,
+        # but for simplicity we load the main yaml structure)
+        main_config_filename = "company.yaml"
+
+        if not self.asset_repo.exists(company_short_name, AssetType.CONFIG, main_config_filename):
+            raise FileNotFoundError(f"Configuration file not found for {company_short_name}")
+
+        yaml_content = self.asset_repo.read_text(company_short_name, AssetType.CONFIG, main_config_filename)
+        config = self.utility.load_yaml_from_string(yaml_content) or {}
+
+        # 2. Update the key in the dictionary
+        self._set_nested_value(config, key, value)
+
+        # 3. Validate the new configuration structure
+        errors = self._validate_configuration(company_short_name, config)
+
+        if errors:
+            logging.warning(f"Configuration update failed validation: {errors}")
+            return config, errors
+
+        # 4. Save back to repository
+        # Assuming Utility has a method to dump YAML. If not, standard yaml library would be needed.
+        # For this example, we assume self.utility.dump_yaml_to_string exists.
+        new_yaml_content = self.utility.dump_yaml_to_string(config)
+        self.asset_repo.write_text(company_short_name, AssetType.CONFIG, main_config_filename, new_yaml_content)
+
+        # 5. Invalidate cache so next reads get the new version
+        if company_short_name in self._loaded_configs:
+            del self._loaded_configs[company_short_name]
+
+        return config, []
+
+    def add_configuration_key(self, company_short_name: str, parent_key: str, key: str, value) -> tuple[dict, list[str]]:
+        """
+        Adds a new key-value pair under a specific parent key in the configuration.
+
+        Args:
+            company_short_name: The company identifier.
+            parent_key: The parent configuration key under which to add the new key (e.g., 'llm').
+            key: The new key name to add.
+            value: The value for the new key.
+
+        Returns:
+            A tuple containing the updated configuration dict and a list of error strings (if any).
         """
-        config_dir = Path("companies") / company_short_name / "config"
-        main_config_path = config_dir / "company.yaml"
+        # 1. Load raw config from file
+        main_config_filename = "company.yaml"
+
+        if not self.asset_repo.exists(company_short_name, AssetType.CONFIG, main_config_filename):
+            raise FileNotFoundError(f"Configuration file not found for {company_short_name}")
+
+        yaml_content = self.asset_repo.read_text(company_short_name, AssetType.CONFIG, main_config_filename)
+        config = self.utility.load_yaml_from_string(yaml_content) or {}
+
+        # 2. Construct full path and set the value
+        # If parent_key is provided, we append the new key to it (e.g., 'llm.new_setting')
+        full_path = f"{parent_key}.{key}" if parent_key else key
+        self._set_nested_value(config, full_path, value)
 
-        if not main_config_path.exists():
-            raise FileNotFoundError(f"Main configuration file not found: {main_config_path}")
+        # 3. Validate the new configuration structure
+        errors = self._validate_configuration(company_short_name, config)
 
-        config = self.utility.load_schema_from_yaml(main_config_path)
+        if errors:
+            logging.warning(f"Configuration add failed validation: {errors}")
+            return config, errors
+
+        # 4. Save back to repository
+        new_yaml_content = self.utility.dump_yaml_to_string(config)
+        self.asset_repo.write_text(company_short_name, AssetType.CONFIG, main_config_filename, new_yaml_content)
+
+        # 5. Invalidate cache
+        if company_short_name in self._loaded_configs:
+            del self._loaded_configs[company_short_name]
+
+        return config, []
+
+
+    def validate_configuration(self, company_short_name: str) -> list[str]:
+        """
+        Public method to trigger validation of the current configuration.
+        """
+        config = self._load_and_merge_configs(company_short_name)
+        return self._validate_configuration(company_short_name, config)
+
+    def _set_nested_value(self, data: dict, key: str, value):
+        """
+        Helper to set a value in a nested dictionary or list using dot notation (e.g. 'llm.model', 'tools.0.name').
+        Handles traversal through both dictionaries and lists.
+        """
+        keys = key.split('.')
+        current = data
+
+        # Traverse up to the parent of the target key
+        for i, k in enumerate(keys[:-1]):
+            if isinstance(current, dict):
+                # If it's a dict, we can traverse or create the path
+                current = current.setdefault(k, {})
+            elif isinstance(current, list):
+                # If it's a list, we MUST use an integer index
+                try:
+                    idx = int(k)
+                    current = current[idx]
+                except (ValueError, IndexError) as e:
+                    raise ValueError(
+                        f"Invalid path: cannot access index '{k}' in list at '{'.'.join(keys[:i + 1])}'") from e
+            else:
+                raise ValueError(
+                    f"Invalid path: '{k}' is not a container (got {type(current)}) at '{'.'.join(keys[:i + 1])}'")
+
+        # Set the final value
+        last_key = keys[-1]
+        if isinstance(current, dict):
+            current[last_key] = value
+        elif isinstance(current, list):
+            try:
+                idx = int(last_key)
+                current[idx] = value
+            except (ValueError, IndexError) as e:
+                raise ValueError(f"Invalid path: cannot assign to index '{last_key}' in list") from e
+        else:
+            raise ValueError(f"Cannot assign value to non-container type {type(current)} at '{key}'")
+
+    def _load_and_merge_configs(self, company_short_name: str) -> dict:
+        """
+        Loads the main company.yaml and merges data from supplementary files
+        specified in the 'content_files' section using AssetRepository.
+        """
+        main_config_filename = "company.yaml"
+
+        # verify existence of the main configuration file
+        if not self.asset_repo.exists(company_short_name, AssetType.CONFIG, main_config_filename):
+            # raise FileNotFoundError(f"Main configuration file not found: {main_config_filename}")
+            logging.exception(f"Main configuration file not found: {main_config_filename}")
+
+            # return the minimal configuration needed for starting the IAToolkit
+            # this is a for solving a chicken/egg problem when trying to migrate the configuration
+            # from filesystem to database in enterprise installation
+            # see create_assets cli command in enterprise-iatoolkit)
+            return {
+                'id': company_short_name,
+                'name': company_short_name,
+                'llm': {'model': 'gpt-5', 'provider_api_keys': {'openai':''} },
+                }
+
+        # read text and parse
+        yaml_content = self.asset_repo.read_text(company_short_name, AssetType.CONFIG, main_config_filename)
+        config = self.utility.load_yaml_from_string(yaml_content)
+        if not config:
+            return {}
 
         # Load and merge supplementary content files (e.g., onboarding_cards)
-        for key, file_path in config.get('help_files', {}).items():
-            supplementary_path = config_dir / file_path
-            if supplementary_path.exists():
-                config[key] = self.utility.load_schema_from_yaml(supplementary_path)
+        for key, filename in config.get('help_files', {}).items():
+            if self.asset_repo.exists(company_short_name, AssetType.CONFIG, filename):
+                supp_content = self.asset_repo.read_text(company_short_name, AssetType.CONFIG, filename)
+                config[key] = self.utility.load_yaml_from_string(supp_content)
             else:
-                logging.warning(f"⚠️  Warning: Content file not found: {supplementary_path}")
-                config[key] = None  # Ensure the key exists but is empty
+                logging.warning(f"⚠️  Warning: Content file not found: {filename}")
+                config[key] = None
 
         return config
 
-    def _register_core_details(self, company_instance, config: dict) -> Company:
+    def _register_company_database(self, config: dict) -> Company:
         # register the company in the database: create_or_update logic
+        if not config:
+            return None
 
-        company_obj = Company(short_name=config['id'],
-                              name=config['name'],
+        # create or update the company in database
+        company_obj = Company(short_name=config.get('id'),
+                              name=config.get('name'),
                               parameters=config.get('parameters', {}))
         company = self.profile_repo.create_company(company_obj)
 
-        # save company object with the instance
-        company_instance.company = company
+        # save company object with the configuration
+        config['company'] = company
+
         return company
 
-    def _register_data_sources(self, company_short_name: str, config: dict):
+    def register_data_sources(self,
+                              company_short_name: str,
+                              config: dict = None):
         """
         Reads the data_sources config and registers databases with SqlService.
         Uses Lazy Loading to avoid circular dependency.
+
+        Public method: Can be called externally after initialization (e.g. by Enterprise)
+        to re-register sources once new factories (like 'bridge') are available.
         """
-        # Lazy import to avoid circular dependency: ConfigService -> SqlService -> I18n -> ConfigService
+
+        # If config is not provided, try to load it from cache
+        if config is None:
+            self._ensure_config_loaded(company_short_name)
+            config = self._loaded_configs.get(company_short_name)
+
+        if not config:
+            return
+
         from iatoolkit import current_iatoolkit
         from iatoolkit.services.sql_service import SqlService
         sql_service = current_iatoolkit().get_injector().get(SqlService)
@@ -129,25 +316,46 @@ class ConfigurationService:
         if not sql_sources:
             return
 
-        logging.info(f"🛢️ Registering databases for '{company_short_name}'...")
-
-        for db_config in sql_sources:
-            db_name = db_config.get('database')
-            db_schema = db_config.get('schema', 'public')
-            db_env_var = db_config.get('connection_string_env')
+        logging.info(f"🛢️ Registering databases  for '{company_short_name}'...")
 
-            # resolve the URI
-            db_uri = os.getenv(db_env_var) if db_env_var else None
+        for source in sql_sources:
+            db_name = source.get('database')
+            if not db_name:
+                continue
 
-            if not db_uri:
+            # Prepare the config dictionary for the factory
+            db_config = {
+                'database': db_name,
+                'schema': source.get('schema', 'public'),
+                'connection_type': source.get('connection_type', 'direct'),
+
+                # Pass through keys needed for Bridge or other plugins
+                'bridge_id': source.get('bridge_id'),
+                'timeout': source.get('timeout')
+            }
+
+            # Resolve URI if env var is present (Required for 'direct', optional for others)
+            db_env_var = source.get('connection_string_env')
+            if db_env_var:
+                db_uri = os.getenv(db_env_var)
+                if db_uri:
+                    db_config['db_uri'] = db_uri
+
+            # Validation: 'direct' connections MUST have a URI
+            if db_config['connection_type'] == 'direct' and not db_config.get('db_uri'):
                 logging.error(
                     f"-> Skipping DB '{db_name}' for '{company_short_name}': missing URI in env '{db_env_var}'.")
                 continue
 
+            elif db_config['connection_type'] == 'bridge' and not db_config.get('bridge_id'):
+                logging.error(
+                    f"-> Skipping DB '{db_name}' for '{company_short_name}': missing bridge_id in configuration.")
+                continue
+
             # Register with the SQL service
-            sql_service.register_database(db_uri, db_name, db_schema)
+            sql_service.register_database(company_short_name, db_name, db_config)
 
-    def _register_tools(self, company_instance, config: dict):
+    def _register_tools(self, company_short_name: str, config: dict):
         """creates in the database each tool defined in the YAML."""
         # Lazy import and resolve ToolService locally
         from iatoolkit import current_iatoolkit
@@ -155,9 +363,9 @@ class ConfigurationService:
         tool_service = current_iatoolkit().get_injector().get(ToolService)
 
         tools_config = config.get('tools', [])
-        tool_service.sync_company_tools(company_instance, tools_config)
+        tool_service.sync_company_tools(company_short_name, tools_config)
 
-    def _register_prompts(self, company_instance, config: dict):
+    def _register_prompts(self, company_short_name: str, config: dict):
         """
          Delegates prompt synchronization to PromptService.
          """
@@ -166,15 +374,27 @@ class ConfigurationService:
         from iatoolkit.services.prompt_service import PromptService
         prompt_service = current_iatoolkit().get_injector().get(PromptService)
 
-        prompts_config = config.get('prompts', [])
-        categories_config = config.get('prompt_categories', [])
+        prompts_config = config.get('prompts', {})
 
         prompt_service.sync_company_prompts(
-            company_instance=company_instance,
-            prompts_config=prompts_config,
-            categories_config=categories_config
+            company_short_name=company_short_name,
+            prompts_config=prompts_config.get('prompt_list', []),
+            categories_config=prompts_config.get('prompt_categories', []),
         )
 
+    def _register_knowledge_base(self, company_short_name: str, config: dict):
+        # Lazy import to avoid circular dependency
+        from iatoolkit import current_iatoolkit
+        from iatoolkit.services.knowledge_base_service import KnowledgeBaseService
+        knowledge_base = current_iatoolkit().get_injector().get(KnowledgeBaseService)
+
+        kb_config = config.get('knowledge_base', {})
+        categories_config = kb_config.get('collections', [])
+
+        # sync collection types in database
+        knowledge_base.sync_collection_types(company_short_name, categories_config)
+
+
     def _validate_configuration(self, company_short_name: str, config: dict):
         """
         Validates the structure and consistency of the company.yaml configuration.
@@ -182,13 +402,15 @@ class ConfigurationService:
         Raises IAToolkitException if any validation error is found.
         """
         errors = []
-        config_dir = Path("companies") / company_short_name / "config"
-        prompts_dir = Path("companies") / company_short_name / "prompts"
 
         # Helper to collect errors
         def add_error(section, message):
             errors.append(f"[{section}] {message}")
 
+        if not config:
+            add_error("General", "Configuration file missing or with errors, check the application logs.")
+            return errors
+
         # 1. Top-level keys
         if not config.get("id"):
             add_error("General", "Missing required key: 'id'")
@@ -204,8 +426,8 @@ class ConfigurationService:
         else:
             if not config.get("llm", {}).get("model"):
                 add_error("llm", "Missing required key: 'model'")
-            if not config.get("llm", {}).get("api-key"):
-                add_error("llm", "Missing required key: 'api-key'")
+            if not config.get("llm", {}).get("provider_api_keys"):
+                add_error("llm", "Missing required key: 'provider_api_keys'")
 
         # 3. Embedding Provider
         if isinstance(config.get("embedding_provider"), dict):
@@ -220,8 +442,12 @@ class ConfigurationService:
         for i, source in enumerate(config.get("data_sources", {}).get("sql", [])):
             if not source.get("database"):
                 add_error(f"data_sources.sql[{i}]", "Missing required key: 'database'")
-            if not source.get("connection_string_env"):
+
+            connection_type = source.get("connection_type")
+            if connection_type == 'direct' and not source.get("connection_string_env"):
                 add_error(f"data_sources.sql[{i}]", "Missing required key: 'connection_string_env'")
+            elif connection_type == 'bridge' and not source.get("bridge_id"):
+                add_error(f"data_sources.sql[{i}]", "Missing bridge_id'")
 
         # 5. Tools
         for i, tool in enumerate(config.get("tools", [])):
@@ -236,15 +462,18 @@ class ConfigurationService:
                 add_error(f"tools[{i}]", "'params' key must be a dictionary.")
 
         # 6. Prompts
-        category_set = set(config.get("prompt_categories", []))
-        for i, prompt in enumerate(config.get("prompts", [])):
+        prompt_list = config.get("prompts", {}).get("prompt_list", [])
+        prompt_categories = config.get("prompts", {}).get("prompt_categories", [])
+
+        category_set = set(prompt_categories)
+        for i, prompt in enumerate(prompt_list):
             prompt_name = prompt.get("name")
             if not prompt_name:
                 add_error(f"prompts[{i}]", "Missing required key: 'name'")
             else:
-                prompt_file = prompts_dir / f"{prompt_name}.prompt"
-                if not prompt_file.is_file():
-                    add_error(f"prompts/{prompt_name}:", f"Prompt file not found: {prompt_file}")
+                prompt_filename = f"{prompt_name}.prompt"
+                if not self.asset_repo.exists(company_short_name, AssetType.PROMPT, prompt_filename):
+                    add_error(f"prompts/{prompt_name}:", f"Prompt file not found: {prompt_filename}")
 
                 prompt_description = prompt.get("description")
                 if not prompt_description:
@@ -289,9 +518,9 @@ class ConfigurationService:
             if not filename:
                 add_error(f"help_files.{key}", "Filename cannot be empty.")
                 continue
-            help_file_path = config_dir / filename
-            if not help_file_path.is_file():
-                add_error(f"help_files.{key}", f"Help file not found: {help_file_path}")
+            if not self.asset_repo.exists(company_short_name, AssetType.CONFIG, filename):
+                add_error(f"help_files.{key}", f"Help file not found: {filename}")
+
 
         # If any errors were found, log all messages and raise an exception
         if errors:
@@ -299,8 +528,5 @@ class ConfigurationService:
                 f" - {e}" for e in errors)
             logging.error(error_summary)
 
-            raise IAToolkitException(
-                IAToolkitException.ErrorType.CONFIG_ERROR,
-                'company.yaml validation errors'
-            )
+        return errors
 

iatoolkit/services/dispatcher_service.py

@@ -59,13 +59,25 @@ class Dispatcher:
         self.setup_iatoolkit_system()
 
         # Loads the configuration of every company: company.yaml file
-        for company_name, company_instance in self.company_instances.items():
+        for company_short_name, company_instance in self.company_instances.items():
             try:
                 # read company configuration from company.yaml
-                self.config_service.load_configuration(company_name, company_instance)
+                config, errors = self.config_service.load_configuration(company_short_name)
+
+                '''
+                if errors:
+                    raise IAToolkitException(
+                        IAToolkitException.ErrorType.CONFIG_ERROR,
+                        'company.yaml validation errors'
+                    )
+                '''
+
+                # complement the instance self data
+                company_instance.company_short_name = company_short_name
+                company_instance.company = config.get('company')
 
             except Exception as e:
-                logging.error(f"❌ Failed to register configuration for '{company_name}': {e}")
+                logging.error(f"❌ Failed to register configuration for '{company_short_name}': {e}")
                 raise e
 
         return True
@@ -95,7 +107,13 @@ class Dispatcher:
 
         # check if action is a system function using ToolService
         if self.tool_service.is_system_tool(function_name):
+            # this is the system function to be executed.
             handler = self.tool_service.get_system_handler(function_name)
+            logging.info(
+                f"Calling system handler [{function_name}] "
+                f"with company_short_name={company_short_name} "
+                f"and kwargs={kwargs}"
+            )
             return handler(company_short_name, **kwargs)
 
         company_instance = self.company_instances[company_short_name]

iatoolkit/services/file_processor_service.py

@@ -74,9 +74,6 @@ class FileProcessor:
                 if not self._apply_filters(file_name):
                     continue
 
-                if self.config.echo:
-                    print(f'loading: {file_name}')
-
                 content = self.connector.get_file_content(file_path)
 
                 # execute the callback function
@@ -87,8 +84,6 @@ class FileProcessor:
                                      context=self.config.context)
                 self.processed_files += 1
 
-                logging.info(f"Successfully processed file: {file_path}")
-
             except Exception as e:
                 logging.error(f"Error processing {file_path}: {e}")
                 if not self.config.continue_on_error: