iatoolkit 1.4.2__py3-none-any.whl → 1.9.0__py3-none-any.whl

iatoolkit/__init__.py

@@ -3,7 +3,7 @@
 #
 # IAToolkit is open source software.
 
-__version__ = "1.4.2"
+__version__ = "1.9.0"
 
 # Expose main classes and functions at the top level of the package
 

iatoolkit/common/interfaces/database_provider.py

@@ -9,14 +9,19 @@ class DatabaseProvider(abc.ABC):
 
     # --- Schema Methods ---
     @abc.abstractmethod
-    def get_all_table_names(self) -> List[str]:
-        pass
-
-    @abc.abstractmethod
-    def get_table_description(self,
-                              table_name: str,
-                              schema_object_name: str | None = None,
-                              exclude_columns: List[str] | None = None) -> str:
+    def get_database_structure(self) -> dict:
+        """
+        Returns the structure of the database (tables, columns, types)
+        Format:
+        {
+            "table_name": {
+                "columns": [
+                    {"name": "col1", "type": "VARCHAR", "nullable": True, "pk": True},
+                    ...
+                ]
+            }
+        }
+        """
         pass
 
     # --- Execution Methods ---

iatoolkit/common/routes.py

@@ -24,7 +24,7 @@ def register_views(app):
     from iatoolkit.views.profile_api_view import UserLanguageApiView
     from iatoolkit.views.embedding_api_view import EmbeddingApiView
     from iatoolkit.views.login_view import LoginView, FinalizeContextView
-    from iatoolkit.views.load_company_configuration_api_view import LoadCompanyConfigurationApiView
+    from iatoolkit.views.configuration_api_view import ConfigurationApiView, ValidateConfigurationApiView
     from iatoolkit.views.logout_api_view import LogoutApiView
     from iatoolkit.views.home_view import HomeView
     from iatoolkit.views.chat_view import ChatView
@@ -32,6 +32,7 @@ def register_views(app):
     from iatoolkit.views.root_redirect_view import RootRedirectView
     from iatoolkit.views.users_api_view import UsersApiView
     from iatoolkit.views.rag_api_view import RagApiView
+    from iatoolkit.views.categories_api_view import CategoriesApiView
 
     # assign root '/' to our new redirect logic
     app.add_url_rule('/home', view_func=RootRedirectView.as_view('root_redirect'))
@@ -85,9 +86,21 @@ def register_views(app):
     # can be used also for executing iatoolkit prompts
     app.add_url_rule('/<company_short_name>/api/llm_query', view_func=LLMQueryApiView.as_view('llm_query_api'))
 
-    # open the promt directory
-    app.add_url_rule('/<company_short_name>/api/prompts', view_func=PromptApiView.as_view('prompt'))
-
+    # Categories Endpoint
+    app.add_url_rule('/<company_short_name>/api/categories',
+                     view_func=CategoriesApiView.as_view('categories_api'),
+                     methods=['GET'])
+
+    # open the promt directory and specific prompt management
+    prompt_view = PromptApiView.as_view('prompt')
+    app.add_url_rule('/<company_short_name>/api/prompts',
+                     view_func=prompt_view,
+                     methods=['GET', 'POST'],
+                     defaults={'prompt_name': None})
+
+    app.add_url_rule('/<company_short_name>/api/prompts/<prompt_name>',
+                     view_func=prompt_view,
+                     methods=['GET', 'POST','PUT', 'DELETE'])
     # toolbar buttons
     app.add_url_rule('/<company_short_name>/api/feedback', view_func=UserFeedbackApiView.as_view('feedback'))
     app.add_url_rule('/<company_short_name>/api/history', view_func=HistoryApiView.as_view('history'))
@@ -128,8 +141,13 @@ def register_views(app):
                      view_func=EmbeddingApiView.as_view('embedding_api'))
 
     # company configuration
-    app.add_url_rule('/<company_short_name>/api/load_configuration',
-                     view_func=LoadCompanyConfigurationApiView.as_view('load-configuration'))
+    app.add_url_rule('/<company_short_name>/api/configuration',
+                     view_func=ConfigurationApiView.as_view('configuration'),
+                     methods=['GET', 'POST', 'PATCH'],)
+
+    app.add_url_rule('/<company_short_name>/api/configuration/validate',
+                     view_func=ValidateConfigurationApiView.as_view('configuration-validate'),
+                     methods=['GET'])
 
     # static pages
     # url: /pages/foundation o /pages/implementation_plan

iatoolkit/common/util.py

@@ -168,6 +168,20 @@ class Utility:
             logging.error(f"Error parsing YAML string: {e}")
             return {}
 
+    def dump_yaml_to_string(self, config: dict) -> str:
+        """
+        Dumps a dictionary into a YAML formatted string.
+        It uses default flow style False to ensure block format (readable YAML).
+        """
+        try:
+            # default_flow_style=False ensures lists and dicts are expanded (not inline like JSON)
+            # allow_unicode=True ensures characters like accents are preserved
+            return yaml.safe_dump(config, default_flow_style=False, allow_unicode=True, sort_keys=False)
+        except yaml.YAMLError as e:
+            logging.error(f"Error dumping YAML to string: {e}")
+            raise IAToolkitException(IAToolkitException.ErrorType.FILE_IO_ERROR,
+                                     f"Failed to generate YAML: {e}")
+
     def generate_context_for_schema(self, entity_name: str, schema_file: str = None, schema: dict = {}) -> str:
         if not schema_file and not schema:
             raise IAToolkitException(IAToolkitException.ErrorType.FILE_IO_ERROR,
@@ -197,9 +211,13 @@ class Utility:
             root_name = list(schema.keys())[0]
             root_details = schema[root_name]
 
+            # support this format
+            if root_details.get('columns'):
+                root_details = root_details['columns']
+
             if isinstance(root_details, dict):
                 # Las claves de metadatos describen el objeto en sí, no sus propiedades hijas.
-                METADATA_KEYS = ['description', 'type', 'format', 'items', 'properties']
+                METADATA_KEYS = ['description', 'type', 'format', 'items', 'properties', 'pk']
 
                 # Las propiedades son las claves restantes en el diccionario.
                 properties = {
@@ -242,6 +260,8 @@ class Utility:
             description = details.get('description', '')
             data_type = details.get('type', 'any')
             output.append(f"{indent_str}- **`{name.lower()}`** ({data_type}): {description}")
+            # if 'pk' in details and details['pk']:
+            #    output.append(f"{indent_str}- **Primary Key**: {details['pk']}")
 
             child_indent_str = '  ' * (indent_level + 1)
 

iatoolkit/infra/connectors/file_connector_factory.py

@@ -40,6 +40,7 @@ class FileConnectorFactory:
             return S3Connector(
                 bucket=config['bucket'],
                 prefix=config.get('prefix', ''),
+                folder=config.get('folder', ''),
                 auth=auth
             )
 

iatoolkit/infra/connectors/s3_connector.py

@@ -9,14 +9,16 @@ from typing import List
 
 
 class S3Connector(FileConnector):
-    def __init__(self, bucket: str, prefix: str, auth: dict):
+    def __init__(self, bucket: str, prefix: str, folder: str, auth: dict):
         self.bucket = bucket
         self.prefix = prefix
+        self.folder = folder
         self.s3 = boto3.client('s3', **auth)
 
     def list_files(self) -> List[dict]:
         # list all the files as dictionaries, with keys:  'path', 'name' y 'metadata'.
-        response = self.s3.list_objects_v2(Bucket=self.bucket, Prefix=self.prefix)
+        prefix = f'{self.prefix}/{self.folder}/'
+        response = self.s3.list_objects_v2(Bucket=self.bucket, Prefix=prefix)
         files = response.get('Contents', [])
 
         return [

iatoolkit/locales/en.yaml

@@ -52,8 +52,7 @@ ui:
     company_config: "Company Configuration (company.yaml)"
     prompts: "Prompts"
     prompts_description: "System Prompts"
-    knowledge: "Knowledge Base"
-    knowledge_rag: "RAG (Vector)"
+    knowledge_rag: "Knowledge Base (RAG)"
     knowledge_static: "Static Context"
     schemas: "Schemas"
     schemas_description: "Data Definitions (YAML)"
@@ -78,7 +77,6 @@ ui:
     saved_ok: "File saved successfully"
     save_file: "Save"
     credentials: "Email and password required for login."
-    saved_ok: "Saved successfully"
     deleted_ok: "Deleted successfully"
     user_manager: "User management"
     admin_page_title: "Admin access"
@@ -87,6 +85,72 @@ ui:
     logout: "Close session"
     error_loading: "Error loading file content"
     loading: "Loading..."
+    add: "Add"
+    create: "Create"
+    delete: "Delete"
+
+
+  db_explorer:
+    data_explorer: "Data Explorer"
+    database_connection: "Database Connection"
+    tables: "Tables"
+    data_explorer_description: "Explore your database tables ."
+    select_database: "Please select a database above to view its tables."
+    not_table_selected: "No table selected"
+    view_yaml: "View YAML"
+    save_schema: "Save schema"
+    select_table: "Select a table from the left panel"
+    table_semantics: "Table Semantics"
+    table_description: "Description (AI Context)"
+    table_placeholder: "Describe what data this table contains (e.g., 'Active and inactive customer records including billing addresses')..."
+    column_metadata: "Column Metadata"
+    auto_detect: "Auto-detected from DB"
+    meta_column: "Column"
+    meta_type: "Type"
+    meta_description: "Description"
+    meta_synonyms: "Synonyms"
+    pii_sesitive: "PII Sensitive"
+
+  prompts:
+    title: "Prompts Manager"
+    subtitle: "Manage and test your AI personas and templates"
+    library: "Prompt Library"
+    new_btn: "New"
+    filter_placeholder: "Filter prompts..."
+    select_prompt: "Select a prompt"
+    toggle_settings: "Toggle Variables & Config"
+    settings_btn: "Settings"
+    save_btn: "Save"
+    tab_editor: "Editor"
+    tab_playground: "Playground"
+    config_title: "Config"
+    desc_label: "Description"
+    desc_placeholder: "Describe this prompt..."
+    vars_label: "Input Variables"
+    no_vars: "No inputs defined"
+    playground_inputs: "Variables & Inputs"
+    no_vars_detected: "No variables detected in prompt template."
+    model_override: "Model Override (Optional)"
+    default_model: "Default (from config)"
+    run_btn: "Run"
+    output_placeholder: "Output will appear here..."
+    new_modal_title: "Create New Prompt"
+    name_label: "Name (Slug)"
+    name_help: "Use lowercase, numbers, and underscores only."
+    category_label: "Category"
+    delete_confirmation: "Delete Prompt?"
+
+
+  config:
+    editor_description: "IAToolkit configuration file"
+    title: "Configuration Editor"
+    sections: "Sections"
+    refresh: "Refresh"
+    validate: "Validate"
+    select_section: "Select a section from the left panel"
+    no_section_selected: "No section selected"
+    view_yaml: "View YAML"
+
 
   rag:
     ingestion: "Ingestion"
@@ -115,8 +179,11 @@ ui:
     delete_message: "This action cannot be undone. The file will be permanently removed."
     delete_button: "Delete"
     delete_cancel: "Cancel"
-
-
+    target_collection: "Collection"
+    select_collection_placeholder: "Select a collection"
+    collection_required: "Collection is required"
+    collection: "Collection"
+    all_collections: "All collections"
 
   tooltips:
     history: "History of my queries"

iatoolkit/locales/es.yaml

@@ -49,8 +49,7 @@ ui:
     workspace: "Recursos"
     configuration: "Configuración"
     company_config: "Configuración Empresa (company.yaml)"
-    knowledge: "Conocimiento"
-    knowledge_rag: "RAG (Vectorial)"
+    knowledge_rag: "Knowledge Base (RAG)"
     knowledge_static: "Contenido Estático"
     prompts: "Prompts"
     prompts_description: "Prompts de sistema"
@@ -62,7 +61,6 @@ ui:
     monitoring: "Monitoreo"
     teams: "Usuarios"
     billing: "Facturación"
-    company: "Empresa"
     files: "Archivos"
     select_file: "Seleccione un archivo a editar"
     select_category: "Seleccione una categoría"
@@ -84,6 +82,70 @@ ui:
     load_configuration: "Guardar configuración"
     goto_chat: "Ir al chat"
     logout: "Cerrar sesión"
+    add: "Agregar"
+    create: "Crear"
+    delete: "Borrar"
+
+  db_explorer:
+    data_explorer: "Explorador de datos"
+    database_connection: "Bases de datos"
+    tables: "Tablas"
+    data_explorer_description: "Explora tus tablas de base de datos."
+    select_database: "Seleccione una base de datos para ver sus tablas."
+    not_table_selected: "No hay tabla seleccionada."
+    view_yaml: "Ver YAML"
+    save_schema: "Guardar esquema"
+    select_table: "Seleccionar una tabla del panel izquierdo"
+    table_semantics: "Significado de la tabla"
+    table_description: "Descripción de la tabla (Contexto IA)"
+    table_placeholder: "Describe el significado de la tabla, ejemplo: tabla de usuarios."
+    column_metadata: "Metadatos de columna"
+    auto_detect: "Auto-detectar desde la BD"
+    meta_column: "Columna"
+    meta_type: "Tipo"
+    meta_description: "Descripción"
+    meta_synonyms: "Sinonimos"
+    pii_sesitive: "IP Sensible"
+
+  prompts:
+    title: "Gestor de Prompts"
+    subtitle: "Gestiona y prueba tus plantillas y personas de IA"
+    library: "Biblioteca de Prompts"
+    new_btn: "Nuevo"
+    filter_placeholder: "Filtrar prompts..."
+    select_prompt: "Selecciona un prompt"
+    toggle_settings: "Alternar Variables y Configuración"
+    settings_btn: "Ajustes"
+    save_btn: "Guardar"
+    tab_editor: "Editor"
+    tab_playground: "Playground"
+    config_title: "Configuración"
+    desc_label: "Descripción"
+    desc_placeholder: "Describe este prompt..."
+    vars_label: "Variables de Entrada"
+    no_vars: "Sin variables definidas"
+    playground_inputs: "Variables y Entradas"
+    no_vars_detected: "No se detectaron variables en la plantilla."
+    model_override: "Modelo Específico (Opcional)"
+    default_model: "Por defecto (según config)"
+    run_btn: "Ejecutar"
+    output_placeholder: "El resultado aparecerá aquí..."
+    new_modal_title: "Crear Nuevo Prompt"
+    name_label: "Nombre (Slug)"
+    name_help: "Solo minúsculas, números y guiones bajos."
+    category_label: "Categoría"
+    delete_confirmation: "Eliminar el prompt?"
+
+  config:
+    editor_description: "Editor de configuración"
+    title: "Editor de configuraciones"
+    sections: "Secciones"
+    refresh: "Refrescar"
+    validate: "Validar"
+    select_section: "Seleccione una sección del panel izquierdo"
+    no_section_selected: "No hay sección seleccionada."
+    view_yaml: "Ver YAML"
+
 
   rag:
     ingestion: "Ingesta"
@@ -101,7 +163,7 @@ ui:
     filename_placeholder: "Contiene..."
     user: "Usuario"
     status: "Estado"
-    all_status: "Todos los estados"
+    all_status: "Estados"
     status_active: "Activo"
     status_pending: "Pendiente"
     status_processing: "Procesando"
@@ -112,6 +174,11 @@ ui:
     delete_message: "Esta acción no se puede deshacer. El archivo se eliminará permanentemente."
     delete_button: "Eliminar"
     delete_cancel: "Cancelar"
+    target_collection: "Categoría"
+    select_collection_placeholder: "Selecciona una categoría"
+    collection_required: "Debe seleccionar una categoría"
+    all_collections: "Todas las categorías"
+    collection: "Categoría"
 
 
   tooltips:

iatoolkit/repositories/database_manager.py

@@ -11,6 +11,7 @@ from iatoolkit.repositories.models import Base
 from injector import inject
 from pgvector.psycopg2 import register_vector
 from iatoolkit.common.interfaces.database_provider import DatabaseProvider
+import logging
 
 
 class DatabaseManager(DatabaseProvider):
@@ -135,51 +136,30 @@ class DatabaseManager(DatabaseProvider):
         self.get_session().rollback()
 
     # -- schema methods ----
-    def get_all_table_names(self) -> list[str]:
-        # Returns a list of all table names in the database
+    def get_database_structure(self) -> dict:
         inspector = inspect(self._engine)
-        return inspector.get_table_names(schema=self.schema)
-
-    def get_table_description(self,
-                              table_name: str,
-                              schema_object_name: str | None = None,
-                              exclude_columns: list[str] | None = None) -> str:
-        inspector = inspect(self._engine)
-
-        # search the table in the specified schema
-        if table_name not in inspector.get_table_names(schema=self.schema):
-            raise RuntimeError(f"Table '{table_name}' does not exist in database schema '{self.schema}'.")
-
-        if exclude_columns is None:
-            exclude_columns = []
-
-        # get all the table columns
-        columns = inspector.get_columns(table_name, schema=self.schema)
-
-        # construct a json dictionary with the table definition
-        json_dict = {
-            "table": table_name,
-            "schema": self.schema,
-            "description": f"The table belongs to the **`{self.schema}`** schema.",
-            "fields": []
-        }
-
-        if schema_object_name:
-            json_dict["description"] += (
-                f"The meaning of each field in this table is detailed in the **`{schema_object_name}`** object."
-            )
-
-        # now add every column to the json dictionary
-        for col in columns:
-            name = col["name"]
-
-            # omit the excluded columns.
-            if name in exclude_columns:
-                continue
-
-            json_dict["fields"].append({
-                "name": name,
-                "type": str(col["type"]),
-            })
-
-        return "\n\n" + str(json_dict)
+        structure = {}
+        for table in inspector.get_table_names(schema=self.schema):
+            columns_data = []
+
+            # get columns
+            try:
+                columns = inspector.get_columns(table, schema=self.schema)
+                # Obtener PKs para marcarlas
+                pks = inspector.get_pk_constraint(table, schema=self.schema).get('constrained_columns', [])
+
+                for col in columns:
+                    columns_data.append({
+                        "name": col['name'],
+                        "type": str(col['type']),
+                        "nullable": col.get('nullable', True),
+                        "pk": col['name'] in pks
+                    })
+            except Exception as e:
+                logging.warning(f"Could not inspect columns for table {table}: {e}")
+
+            structure[table] = {
+                "columns": columns_data
+            }
+
+        return structure

iatoolkit/repositories/llm_query_repo.py

@@ -3,10 +3,13 @@
 #
 # IAToolkit is open source software.
 
-from iatoolkit.repositories.models import LLMQuery, Tool, Company, Prompt, PromptCategory
+from iatoolkit.repositories.models import (LLMQuery, Tool,
+                    Company, Prompt, PromptCategory, PromptType)
 from injector import inject
 from iatoolkit.repositories.database_manager import DatabaseManager
 from sqlalchemy import or_
+from typing import List
+
 
 class LLMQueryRepo:
     @inject
@@ -54,7 +57,7 @@ class LLMQueryRepo:
             self.session.add(new_tool)
             tool = new_tool
 
-        self.session.flush()
+        self.session.commit()
         return tool
 
     def delete_tool(self, tool: Tool):
@@ -67,14 +70,14 @@ class LLMQueryRepo:
             prompt.category_id = new_prompt.category_id
             prompt.description = new_prompt.description
             prompt.order = new_prompt.order
-            prompt.is_system_prompt = new_prompt.is_system_prompt
+            prompt.prompt_type = new_prompt.prompt_type
             prompt.filename = new_prompt.filename
             prompt.custom_fields = new_prompt.custom_fields
         else:
             self.session.add(new_prompt)
             prompt = new_prompt
 
-        self.session.flush()
+        self.session.commit()
         return prompt
 
     def create_or_update_prompt_category(self, new_category: PromptCategory):
@@ -94,12 +97,31 @@ class LLMQueryRepo:
             LLMQuery.user_identifier == user_identifier,
         ).filter_by(company_id=company.id).order_by(LLMQuery.created_at.desc()).limit(100).all()
 
-    def get_prompts(self, company: Company) -> list[Prompt]:
-        return self.session.query(Prompt).filter_by(company_id=company.id, is_system_prompt=False).all()
+    def get_prompts(self, company: Company, include_all: bool = False) -> list[Prompt]:
+        if include_all:
+            # Include all  prompts: company, system, agent
+            return self.session.query(Prompt).filter(
+                Prompt.company_id == company.id,
+            ).all()
+        else:
+            # Only company prompts, excluding system (default behavior for end users)
+            return self.session.query(Prompt).filter(
+                Prompt.company_id == company.id,
+                Prompt.prompt_type == PromptType.COMPANY.value
+            ).all()
 
     def get_prompt_by_name(self, company: Company, prompt_name: str):
         return self.session.query(Prompt).filter_by(company_id=company.id, name=prompt_name).first()
 
+    def get_category_by_name(self, company_id: int, name: str) -> PromptCategory:
+        return self.session.query(PromptCategory).filter_by(company_id=company_id, name=name).first()
+
+    def get_all_categories(self, company_id: int) -> List[PromptCategory]:
+        return self.session.query(PromptCategory).filter_by(company_id=company_id).order_by(PromptCategory.order).all()
+
     def get_system_prompts(self) -> list[Prompt]:
-        return self.session.query(Prompt).filter_by(is_system_prompt=True, active=True).order_by(Prompt.order).all()
+        return self.session.query(Prompt).filter_by(prompt_type=PromptType.SYSTEM.value, active=True).order_by(Prompt.order).all()
 
+    def delete_prompt(self, prompt: Prompt):
+        self.session.delete(prompt)
+        self.session.commit()

iatoolkit/repositories/models.py

@@ -17,6 +17,19 @@ import enum
 class Base(DeclarativeBase):
     pass
 
+
+class DocumentStatus(str, enum.Enum):
+    PENDING = "pending"
+    PROCESSING = "processing"
+    ACTIVE = "active"
+    FAILED = "failed"
+
+class PromptType(str, enum.Enum):
+    SYSTEM = "system"
+    COMPANY = "company"
+    AGENT = "agent"
+
+
 # relation table for many-to-many relationship between companies and users
 user_company = Table('iat_user_company',
                      Base.metadata,
@@ -149,11 +162,6 @@ class Tool(Base):
         return {column.key: getattr(self, column.key) for column in class_mapper(self.__class__).columns}
 
 
-class DocumentStatus(str, enum.Enum):
-    PENDING = "pending"
-    PROCESSING = "processing"
-    ACTIVE = "active"
-    FAILED = "failed"
 
 
 class CollectionType(Base):
@@ -290,12 +298,13 @@ class Prompt(Base):
     description = Column(String, nullable=False)
     filename = Column(String, nullable=False)
     active = Column(Boolean, default=True)
-    is_system_prompt = Column(Boolean, default=False)
+    prompt_type = Column(String, default=PromptType.COMPANY.value, nullable=False)
     order = Column(Integer, nullable=True, default=0)
     category_id = Column(Integer, ForeignKey('iat_prompt_categories.id'), nullable=True)
     custom_fields = Column(JSON, nullable=False, default=[])
-
     created_at = Column(DateTime, default=datetime.now)
+    def to_dict(self):
+        return {column.key: getattr(self, column.key) for column in class_mapper(self.__class__).columns}
 
     company = relationship("Company", back_populates="prompts")
     category = relationship("PromptCategory", back_populates="prompts")