iatoolkit 2.74.3__tar.gz → 2.75.1__tar.gz

{iatoolkit-2.74.3/src/iatoolkit.egg-info → iatoolkit-2.75.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iatoolkit
-Version: 2.74.3
+Version: 2.75.1
 Summary: IAToolkit
 Author: Fernando Libedinsky
 License-Expression: MIT

{iatoolkit-2.74.3 → iatoolkit-2.75.1}/src/iatoolkit/__init__.py

@@ -3,7 +3,7 @@
 #
 # IAToolkit is open source software.
 
-__version__ = "2.74.3"
+__version__ = "2.75.1"
 
 # Expose main classes and functions at the top level of the package
 

{iatoolkit-2.74.3 → iatoolkit-2.75.1}/src/iatoolkit/config/system_prompts/sql_core.prompt

@@ -3,10 +3,13 @@
 - Trabaja siempre con el dialecto SQL indicado para la base seleccionada.
 - Antes de escribir la consulta, revisa el contexto de la base y respeta su marcador
   `dialect=...` (por ejemplo `postgresql` o `mysql`).
-- Usa solo tablas, columnas y relaciones presentes en el schema provisto.
+- El contexto lista las bases y sus tablas, sin columnas. Antes de escribir la
+  consulta, pide con `iat_sql_schema` las columnas de las tablas que vas a usar,
+  todas en una sola llamada. Si una columna no aparece, pide la tabla que la tiene
+  en vez de suponerla.
+- Usa solo tablas de la lista y columnas que `iat_sql_schema` te haya mostrado.
   No inventes columnas ni supongas nombres parecidos.
-- Si la consulta requiere una base especifica, usa `iat_sql_query` con el
-  `database_key` correcto.
+- Ejecuta la consulta con `iat_sql_query` y el `database_key` correcto.
 - Si hay mas de una tabla, subconsulta o CTE, califica las columnas con alias
   en `SELECT`, `JOIN`, `ON`, `WHERE`, `GROUP BY`, `HAVING` y `ORDER BY`.
 - No mezcles sintaxis de distintos dialectos en la misma consulta.

{iatoolkit-2.74.3 → iatoolkit-2.75.1}/src/iatoolkit/config/system_tools_pack.yaml

@@ -220,6 +220,39 @@ tools:
       required:
         - database_key
         - query
+  - function_name: iat_sql_schema
+    description: "Columnas de tablas SQL: antes de escribir una consulta para iat_sql_query, pide aquí las columnas de las tablas que vas a usar. El contexto trae la lista de bases y tablas; esta herramienta trae sus columnas, tipos y descripciones. Pide varias tablas en una sola llamada."
+    routing:
+      force_include_capability: has_sql_sources
+    routing_profile:
+      tags:
+        - sql
+        - database
+        - schema
+        - columns
+      intents:
+        - Get the columns of SQL tables before writing a query.
+        - Inspect the structure of a configured SQL database.
+      examples:
+        - iat_sql_schema(database_key="erp", table_names=["customers", "orders"])
+      cost:
+        penalty: 0.05
+        latency_class: low
+        expensive: false
+    parameters:
+      type: object
+      properties:
+        database_key:
+          type: string
+          description: "Nombre exacto de la base, como aparece en el contexto."
+        table_names:
+          type: array
+          items:
+            type: string
+          description: "Tablas de esa base cuyas columnas necesitas (hasta 10)."
+      required:
+        - database_key
+        - table_names
   - function_name: iat_document_search
     description: Búsqueda semántica sobre documentos internos de la empresa para traer contexto adicional cuando haga falta. No usar si el usuario pide trabajar exclusivamente con un archivo ya adjunto en el turno actual. Usa metadata_filter solo cuando el usuario o el contexto indiquen un filtro de metadata concreto y necesario; si no hace falta, envía null.
     routing:

{iatoolkit-2.74.3 → iatoolkit-2.75.1}/src/iatoolkit/services/company_context_service.py

@@ -65,7 +65,6 @@ class CompanyContextService:
         sql_context = ""
         yaml_context = ""
         db_tables = []
-        sql_source_table_scopes = {}
 
         # 1. Context from Markdown (context/*.md)  files
         try:
@@ -85,7 +84,6 @@ class CompanyContextService:
                                                                         allowed_databases=allowed_databases,
                                                                         table_filter=table_filter)
                 sql_context = sql_context.strip()
-                sql_source_table_scopes = self._get_sql_source_table_scopes(company_short_name)
             except Exception as e:
                 logging.warning(f"Could not generate SQL context for '{company_short_name}': {e}")
 
@@ -95,7 +93,6 @@ class CompanyContextService:
                 yaml_schema_context = self._get_yaml_schema_context(
                     company_short_name,
                     db_tables,
-                    sql_source_table_scopes=sql_source_table_scopes,
                 ).strip()
                 if yaml_schema_context:
                     yaml_context = f"## Esquemas adicionales\n\n{yaml_schema_context}"
@@ -153,17 +150,6 @@ class CompanyContextService:
             logging.warning(f"Could not generate SQL context for '{company_short_name}': {e}")
             return ""
 
-    def _get_sql_source_table_scopes(self, company_short_name: str) -> dict[str, set[str] | None]:
-        scopes: dict[str, set[str] | None] = {}
-        sql_sources = self.sql_source_service.list_sources(company_short_name, include_inactive=False)
-
-        for source in sql_sources:
-            db_name = str(source.get("database") or "").strip()
-            if not db_name:
-                continue
-            scopes[db_name] = self._resolve_allowed_tables_for_source(source)
-        return scopes
-
     def _resolve_allowed_tables_for_source(self, source: dict) -> set[str] | None:
         if bool(source.get("include_all_tables", True)):
             return None
@@ -181,160 +167,59 @@ class CompanyContextService:
             return False
         return bool(cls._table_name_candidates(table_name) & allowed_tables)
 
+    @property
+    def sql_catalog(self):
+        """What of each SQL source a person sees (BL-017); built here, it reads this service's schema."""
+        if getattr(self, "_sql_catalog", None) is None:
+            from iatoolkit.services.sql_catalog_service import SqlCatalogService
+
+            self._sql_catalog = SqlCatalogService(self, self.sql_source_service, self.sql_service)
+        return self._sql_catalog
+
     def _get_sql_enriched_context(self, company_short_name: str, allowed_databases: List[str] | None = None,
                                   table_filter=None):
         """
-        Generates the SQL context for the LLM using the enriched schema logic.
-        It iterates over configured databases, fetches their enriched structure,
-        and formats it into a prompt-friendly string.
-        """
-        sql_sources = self.sql_source_service.list_sources(company_short_name, include_inactive=False)
-        if not sql_sources:
-            return '', []
-
-        allowed_database_set = {
-            str(database or "").strip()
-            for database in (allowed_databases or [])
-            if str(database or "").strip()
-        }
-        if allowed_databases is not None and not allowed_database_set:
-            return "", []
-
-        context_output = []
-        db_tables=[]
-
-        for source in sql_sources:
-            db_name = source.get('database')
-            if not db_name:
-                continue
-            if allowed_database_set and db_name not in allowed_database_set:
-                continue
-
-            allowed_tables = self._resolve_allowed_tables_for_source(source)
-
-            try:
-                # 1. Get the Enriched Schema (Physical + YAML)
-                enriched_structure = self.get_enriched_database_schema(company_short_name, db_name)
-                if not enriched_structure:
-                    continue
-
-                if allowed_tables is not None:
-                    enriched_structure = {
-                        table_name: table_data
-                        for table_name, table_data in enriched_structure.items()
-                        if self._table_is_allowed(table_name, allowed_tables)
-                    }
-                    if not enriched_structure:
-                        continue
-                if table_filter is not None:
-                    # Tables the person may not query (enterprise's knowledge tables, BL-010).
-                    enriched_structure = {
-                        table_name: table_data
-                        for table_name, table_data in enriched_structure.items()
-                        if table_filter(db_name, table_name)
-                    }
-                    if not enriched_structure:
-                        continue
-
-                # 2. Build Header for this Database
-                dialect = self.sql_service.get_database_dialect(company_short_name, db_name)
-                dialect_suffix = f" [dialect={dialect}]" if dialect else ""
-                db_context = f"### Base SQL (`database_key`): {db_name}{dialect_suffix}\n"
-
-                # Optional: Add DB description from config if available (useful context)
-                db_desc = source.get('description', '')
-                if db_desc:
-                    db_context += f"- Descripcion: {db_desc}\n"
-
-                db_context += (
-                    f"- Usa `iat_sql_query` con `database_key='{db_name}'` para consultar esta base.\n"
-                )
-
-                # 3. Format Tables
-                for table_name, table_data in enriched_structure.items():
-                    table_desc = table_data.get('description', '')
-                    columns = table_data.get('columns', [])
-
-                    # Table Header
-                    table_str = f"\n\n#### Tabla: `{table_name}`"
-                    if table_desc:
-                        table_str += f"\n- Descripcion: {table_desc}"
+        The SQL block of the base context: a map of each source the person may
+        query, its tables with a one-line description and no columns (BL-017).
 
-                    table_str += "\n- Columnas:"
-
-                    # Format Columns
-                    for col in columns:
-                        col_name = col.get('name')
-                        col_type = col.get('type', 'unknown')
-                        col_desc = col.get('description', '')
-                        col_props = col.get('properties') # Nested JSONB structure
-
-                        col_line = f"\n  - `{col_name}` ({col_type})"
-                        if col_desc:
-                            col_line += f": {col_desc}"
-
-                        table_str += col_line
-
-                        # If it has nested properties (JSONB enriched from YAML), format them
-                        if col_props:
-                            table_str += "\n"
-                            table_str += self._format_json_schema(col_props, 2) # Indent level 2
-
-                    db_context += table_str
-
-                    # collect the table names for later use
-                    db_tables.append(
-                        {'db_name': db_name,
-                         'table_name': table_name,
-                         }
-                    )
-
-                context_output.append(db_context)
-
-            except Exception as e:
-                logging.warning(f"Could not generate enriched SQL context for '{db_name}': {e}")
-
-        if not context_output:
-            return "", []
-
-        header = (
-            "## Bases de datos SQL disponibles\n\n"
-            "Estas son las bases SQL que puedes consultar usando `iat_sql_query`. "
-            "El schema a continuacion incluye metadata enriquecida.\n"
+        The columns travel only when a question needs them, through
+        `iat_sql_schema`: the whole schema in the base context was resent on
+        every model call and never compacted (~33k tokens at Maxxa).
+        Returns the block and the (db_name, table_name) pairs it covers.
+        """
+        sources = self.sql_catalog.sources(
+            company_short_name, allowed_databases=allowed_databases, table_filter=table_filter,
         )
-        return header + "\n\n---\n\n".join(context_output), db_tables
-
+        db_tables = [
+            {"db_name": source["database_key"], "table_name": table["name"]}
+            for source in sources
+            for table in source["tables"]
+        ]
+        return self.sql_catalog.render_map(sources), db_tables
 
     def _get_yaml_schema_context(
         self,
         company_short_name: str,
         db_tables: List[Dict],
-        sql_source_table_scopes: dict[str, set[str] | None] | None = None,
     ) -> str:
         # Get context from .yaml schema files using the repository
         yaml_schema_context = ''
-        sql_source_table_scopes = sql_source_table_scopes or {}
 
         try:
             # 1. List yaml files in the schema "folder"
             schema_files = self.asset_repo.list_files(company_short_name, AssetType.SCHEMA, extension='.yaml')
 
+            # A `<db>-<table>.yaml` of a configured SQL source belongs to the SQL
+            # schema, never to this block: it reaches the model through the
+            # catalogue (map + iat_sql_schema), which applies every filter. Read
+            # here, a table hidden from a person (a knowledge table they may not
+            # use) came back whole as an "additional schema".
+            sql_source_names = self._configured_sql_source_names(company_short_name)
+
             for filename in schema_files:
-                # skip tables that are already in the SQL context
                 if '-' in filename:
-                    dbname, f = filename.split("-", 1)
-                    table_name = f.split('.')[0]
-
-                    exists = any(
-                        item["db_name"] == dbname
-                        and bool(self._table_name_candidates(item["table_name"]) & self._table_name_candidates(table_name))
-                        for item in db_tables
-                    )
-                    if exists:
-                        continue
-
-                    allowed_tables = sql_source_table_scopes.get(dbname)
-                    if dbname in sql_source_table_scopes and not self._table_is_allowed(table_name, allowed_tables):
+                    dbname = self._normalize_identifier(filename.split("-", 1)[0])
+                    if dbname in sql_source_names:
                         continue
 
                 try:
@@ -357,6 +242,15 @@ class CompanyContextService:
 
         return yaml_schema_context
 
+    def _configured_sql_source_names(self, company_short_name: str) -> set[str]:
+        """Every SQL source of the company, active or not, by normalized name."""
+        try:
+            sources = self.sql_source_service.list_sources(company_short_name, include_inactive=True) or []
+        except Exception as e:
+            logging.warning(f"Could not list the SQL sources of '{company_short_name}': {e}")
+            return set()
+        return {self._normalize_identifier(s.get("database")) for s in sources if s.get("database")}
+
     def generate_schema_table(self, schema: dict) -> str:
         if not schema or not isinstance(schema, dict):
             return ""

{iatoolkit-2.74.3 → iatoolkit-2.75.1}/src/iatoolkit/services/context_builder_service.py

@@ -403,8 +403,10 @@ class ContextBuilderService:
             company_short_name,
             allowed_databases=allowed_databases,
             table_filter=access.sql_table_filter(company_short_name, user_identifier),
-            include_sql_context=bool(company_context_policy.get("sql_context")),
-            include_yaml_context=bool(company_context_policy.get("yaml_context")),
+            # Only the Markdown block reaches an agent; its SQL is its bound sources
+            # (sql_context above), never the global map or the residual YAML.
+            include_sql_context=False,
+            include_yaml_context=False,
         )
         markdown_context = (
             company_context_blocks.get("markdown_context")

{iatoolkit-2.74.3 → iatoolkit-2.75.1}/src/iatoolkit/services/dispatcher_service.py

@@ -33,6 +33,8 @@ class Dispatcher:
         "iat_memory_answer_question",
         "iat_wiki_search",
         "iat_wiki_get_page",
+        # The schema shown is the person's: their sources and tables (BL-017).
+        "iat_sql_schema",
     }
     _tool_execution_hook = None
     # Tool types this module executes itself; an extension cannot take them over.

{iatoolkit-2.74.3 → iatoolkit-2.75.1}/src/iatoolkit/services/llm_client_service.py

@@ -1606,8 +1606,9 @@ class llmClient:
         sql {sql_query}
 
         **INSTRUCCIÓN OBLIGATORIA:**
-        1.  Analiza el error y corrige la sintaxis de la consulta SQL anterior.
-        2.  Llama a la herramienta `{function_name}` **OTRA VEZ**, inmediatamente, con la consulta corregida.
+        1.  Analiza el error y corrige la consulta SQL anterior. Si dice que una columna o una tabla
+            no existe, pide primero sus columnas reales con `iat_sql_schema`; no adivines otro nombre.
+        2.  Llama a la herramienta `{function_name}` **OTRA VEZ** con la consulta corregida.
         3.  **NO** respondas al usuario con este mensaje de error. Tu ÚNICA acción debe ser volver a llamar a la herramienta con la solución.
         """
 

{iatoolkit-2.74.3 → iatoolkit-2.75.1}/src/iatoolkit/services/prompt_service.py

@@ -62,10 +62,11 @@ class PromptService:
     AGENT_ROLE_CHANNELS = PromptAgentRole.CHANNELS.value
     AGENT_ROLE_OPERATIONS = PromptAgentRole.OPERATIONS.value
     DEFAULT_CATEGORY_LABEL = "General"
+    # An agent's SQL comes only from the sources bound to it: "Global SQL" and
+    # "YAML" were removed, neither ever reached the agent's context. A stored
+    # policy that still names sql_context or yaml_context loads; the key is ignored.
     COMPANY_CONTEXT_BLOCK_KEYS = {
         "markdown_context",
-        "sql_context",
-        "yaml_context",
     }
 
     @inject
@@ -187,14 +188,10 @@ class PromptService:
         if normalized == cls.AGENT_ROLE_WORKSPACE_CHAT:
             blocks = {
                 "markdown_context": True,
-                "sql_context": True,
-                "yaml_context": True,
             }
         else:
             blocks = {
                 "markdown_context": True,
-                "sql_context": False,
-                "yaml_context": False,
             }
         return {
             "company_context_blocks": blocks,

{iatoolkit-2.74.3 → iatoolkit-2.75.1}/src/iatoolkit/services/resource_access_scope.py

@@ -32,7 +32,16 @@ from iatoolkit.common.exceptions import IAToolkitException
 from iatoolkit.repositories.database_manager import DatabaseManager
 from iatoolkit.repositories.models import CollectionType, Company, Document, KnowledgeWiki
 
-SQL_TOOLS = frozenset({"iat_sql_query"})
+SQL_TOOLS = frozenset({"iat_sql_query", "iat_sql_schema"})
+#: Tools whose access follows another tool's (BL-017): whoever may query may read
+#: the schema of what they query, and nobody else. Without this a new tool falls
+#: to the type default and can end up wider or narrower than the one it serves.
+TOOL_ACCESS_ALIASES = {"iat_sql_schema": "iat_sql_query"}
+
+
+def tool_access_key(function_name: str) -> str:
+    """The tool name whose access list decides this tool's access."""
+    return TOOL_ACCESS_ALIASES.get(function_name, function_name)
 COLLECTION_TOOLS = frozenset({"iat_document_search", "iat_document_list", "iat_image_search", "iat_visual_search"})
 DOCUMENT_READ_TOOLS = frozenset({"iat_document_read"})
 WIKI_TOOLS = frozenset({"iat_wiki_search", "iat_wiki_get_page"})
@@ -55,7 +64,7 @@ class ResourceAccessScope:
         """The call's arguments, narrowed to what the person may reach; raises PERMISSION otherwise."""
         if not user_identifier:
             return arguments
-        if not access.can(company_short_name, user_identifier, USE, "tool", function_name):
+        if not access.can(company_short_name, user_identifier, USE, "tool", tool_access_key(function_name)):
             raise _denied(f"No access to the tool '{function_name}'.")
 
         scoped = dict(arguments)