iatoolkit 2.75.4__tar.gz → 2.76.1__tar.gz

{iatoolkit-2.75.4/src/iatoolkit.egg-info → iatoolkit-2.76.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iatoolkit
-Version: 2.75.4
+Version: 2.76.1
 Summary: IAToolkit
 Author: Fernando Libedinsky
 License-Expression: MIT

{iatoolkit-2.75.4 → iatoolkit-2.76.1}/src/iatoolkit/__init__.py

@@ -3,7 +3,7 @@
 #
 # IAToolkit is open source software.
 
-__version__ = "2.75.4"
+__version__ = "2.76.1"
 
 # Expose main classes and functions at the top level of the package
 

{iatoolkit-2.75.4 → iatoolkit-2.76.1}/src/iatoolkit/config/system_tools_pack.yaml

@@ -666,6 +666,35 @@ tools:
       required:
         - question_id
         - answer
+  - function_name: iat_memory_register_question
+    description: Registra una pregunta sobre cómo se hacen las cosas en la empresa que la Memoria de la empresa no pudo responder, para que alguien que sabe la responda y la Memoria lo aprenda. Úsala cuando, después de buscar en la Memoria y en los documentos, no encontraste la respuesta, o cuando el usuario pida agregar la pregunta a la Memoria. No guarda quién preguntó. No la uses para preguntas personales, del momento o que no son de la empresa. Cuéntale al usuario que quedó registrada y qué pasa después (o que no se registró, si así lo dice el resultado).
+    execution_config:
+      side_effects: true
+      destructive: false
+    routing_profile:
+      tags:
+        - memory
+        - company_memory
+        - questions
+        - unanswered
+        - gap
+      intents:
+        - Register a question about the company that the memory could not answer.
+        - Add the user's question to the company memory's open questions.
+      examples:
+        - iat_memory_register_question(question="¿Cuál es el procedimiento para emitir una factura en euros?")
+      cost:
+        penalty: 0.1
+        latency_class: low
+        expensive: false
+    parameters:
+      type: object
+      properties:
+        question:
+          type: string
+          description: La pregunta, redactada en tono neutro y autocontenida (debe entenderse sin la conversación), sin nombres ni datos personales.
+      required:
+        - question
   - function_name: iat_image_search
     description: Busca imágenes en la base de conocimiento visual de la empresa usando una descripción de texto. Útil cuando el usuario pide 'ver' algo, 'muéstrame una foto de...', o busca gráficos y diagramas.
     routing:

{iatoolkit-2.75.4 → iatoolkit-2.76.1}/src/iatoolkit/services/dispatcher_service.py

@@ -31,6 +31,7 @@ class Dispatcher:
         "iat_memory_save",
         "iat_memory_pending_questions",
         "iat_memory_answer_question",
+        "iat_memory_register_question",
         "iat_wiki_search",
         "iat_wiki_get_page",
         # The schema shown is the person's: their sources and tables (BL-017).

{iatoolkit-2.75.4 → iatoolkit-2.76.1}/src/iatoolkit/services/history_compaction_service.py

@@ -300,6 +300,9 @@ class HistoryCompactionService:
                 return {"status": "skipped", "reason": "history_changed"}
 
             compacted = [current[0], HistoryManagerService.summary_message(summary)] + current[plan.cut:]
+            # The turns kept verbatim keep their words, not their old tool
+            # outputs: else a compaction can leave the history as big as it was.
+            HistoryManagerService._trim_old_tool_outputs(compacted)
             self.session_context.save_context_history(company_short_name, user_identifier, compacted, model=model)
             # The provider's last measure described the long history; until the
             # next turn measures again, an estimate keeps the in-line check from

{iatoolkit-2.75.4 → iatoolkit-2.76.1}/src/iatoolkit/services/history_manager_service.py

@@ -64,8 +64,10 @@ class HistoryManagerService:
     #: stored history: an SQL result or a schema is what the model needs on the
     #: turn that asked for it, and resending it whole on every later turn was
     #: most of the history's growth (4k -> 11k tokens on one SQL turn in
-    #: production). The model can call the tool again.
-    KEEP_TOOL_OUTPUT_TURNS = 2
+    #: production). The model can call the tool again. One: the last turn, the
+    #: one a follow-up ("of those...") refers to; with two, one turn of 80k of
+    #: results kept a compacted history at 130k.
+    KEEP_TOOL_OUTPUT_TURNS = 1
     #: What is kept of a cut output: its beginning, enough to remember its shape.
     TRIMMED_TOOL_OUTPUT_CHARS = 1500
     TRIMMED_TOOL_OUTPUT_MARKER = "[tool output trimmed"
@@ -388,13 +390,19 @@ class HistoryManagerService:
 
     @classmethod
     def _trim_old_tool_outputs(cls, context_history: list):
-        """Cuts the long tool outputs of all but the last KEEP_TOOL_OUTPUT_TURNS turns, in place.
-
-        Each output keeps its message (a tool call without its output is a
-        history every provider rejects) and its beginning, plus a note naming the
-        tool so the model knows it can call it again. An output is cut once: the
-        stored history then stays the same from turn to turn, which is what a
-        provider's prompt cache reuses.
+        """Cuts what the turns before the last KEEP_TOOL_OUTPUT_TURNS carry for their tools, in place.
+
+        - Long tool outputs keep their beginning and a note naming the tool, so
+          the model knows it can call it again.
+        - Long tool call arguments (a whole dataset passed to an Excel, for
+          example) become a short valid JSON object: some adapters parse them.
+        - The reasoning of those turns is dropped: providers need it only within
+          the turn that produced it, and 8 rounds of it stayed in a compacted
+          history (110k tokens one turn after compacting, in production).
+
+        Every call keeps its message and its output message, so the history is
+        always valid for the provider. A message is cut once: the stored history
+        then stays the same from turn to turn, which is what a prompt cache reuses.
         """
         turn_starts = cls.turn_starts(context_history)
         if len(turn_starts) <= cls.KEEP_TOOL_OUTPUT_TURNS:
@@ -402,9 +410,19 @@ class HistoryManagerService:
         boundary = turn_starts[-cls.KEEP_TOOL_OUTPUT_TURNS]
         tool_names = {}
         for message in context_history[:boundary]:
-            for call in (message.get("tool_calls") or []) if isinstance(message, dict) else []:
-                if isinstance(call, dict):
-                    tool_names[call.get("id")] = (call.get("function") or {}).get("name")
+            if not isinstance(message, dict):
+                continue
+            message.pop("reasoning_content", None)
+            message.pop("reasoning_details", None)
+            for call in message.get("tool_calls") or []:
+                if not isinstance(call, dict):
+                    continue
+                function = call.get("function") if isinstance(call.get("function"), dict) else {}
+                tool_names[call.get("id")] = function.get("name")
+                arguments = function.get("arguments")
+                if isinstance(arguments, str) and len(arguments) > cls.TRIMMED_TOOL_OUTPUT_CHARS:
+                    function["arguments"] = json.dumps(
+                        {"trimmed": f"{len(arguments)} characters of arguments, from an earlier turn"})
         for message in context_history[:boundary]:
             if not isinstance(message, dict) or message.get("type") != "function_call_output":
                 continue

{iatoolkit-2.75.4 → iatoolkit-2.76.1}/src/iatoolkit/services/llm_client_service.py

@@ -66,6 +66,16 @@ class llmClient:
     # a wiki page withheld).
     TOOL_CALL_RESULT_KEY = "result"
     TOOL_CALL_EXCERPT_MAX_LENGTH = 300
+    # The most of one tool output the model reads (~10k tokens). Every round of
+    # the loop resends every output, so a few results of hundreds of rows took
+    # one turn from 54k to 133k tokens in production (BL-009). Past this, the
+    # model is told to narrow the query instead.
+    MAX_TOOL_OUTPUT_CHARS = 40000
+    TOOL_OUTPUT_CUT_NOTICE = (
+        "[Resultado cortado: se muestran {shown} de {total} caracteres. No lo completes ni "
+        "lo inventes: si necesitas el resto, vuelve a llamar la herramienta pidiendo menos "
+        "(filtros, agregaciones, LIMIT o por partes).]"
+    )
     TOOL_RESULT_FAILED_STATUSES = frozenset({"error", "auth_required", "forbidden", "unavailable"})
 
     @inject
@@ -408,7 +418,8 @@ class llmClient:
                             tool_attachments_count = len(tool_native_attachments or [])
 
                             # add the return value into the list of messages
-                            tool_output_text = self._serialize_tool_output(result)
+                            tool_output_text = self._cap_tool_output(
+                                company.short_name, function_name, self._serialize_tool_output(result))
                             last_tool_output_message = {
                                 "type": "function_call_output",
                                 "call_id": tool_call.call_id,
@@ -774,6 +785,17 @@ class llmClient:
         except Exception:
             return str(result)
 
+    @classmethod
+    def _cap_tool_output(cls, company_short_name: str, function_name: str, text: str) -> str:
+        if not isinstance(text, str) or len(text) <= cls.MAX_TOOL_OUTPUT_CHARS:
+            return text
+        logging.warning(
+            f"[{company_short_name}] output of {function_name} cut to {cls.MAX_TOOL_OUTPUT_CHARS} "
+            f"of {len(text)} characters."
+        )
+        notice = cls.TOOL_OUTPUT_CUT_NOTICE.format(shown=cls.MAX_TOOL_OUTPUT_CHARS, total=len(text))
+        return f"{text[:cls.MAX_TOOL_OUTPUT_CHARS]}\n{notice}"
+
     @staticmethod
     def _split_tool_result_and_native_attachments(result):
         if not isinstance(result, dict):

{iatoolkit-2.75.4 → iatoolkit-2.76.1}/src/iatoolkit/services/memory_question_service.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 
 import json
 import logging
+import re
 from contextlib import contextmanager
 from datetime import datetime, timedelta
 
@@ -206,6 +207,85 @@ class MemoryQuestionService:
         except Exception:
             logging.exception("Memoria search miss could not be recorded for %s", company_short_name)
 
+    # =====================================================================
+    # A question someone says the Memoria does not answer
+    # =====================================================================
+
+    # Words that say nothing about the topic of a question.
+    QUESTION_STOPWORDS = frozenset({
+        "como", "cual", "cuales", "cuando", "cuanto", "cuanta", "cuantos", "cuantas", "donde", "quien", "quienes",
+        "para", "pero", "porque", "sobre", "entre", "desde", "hasta", "esta", "este", "estos", "estas", "esto",
+        "puedo", "puede", "pueden", "podemos", "debo", "debe", "tiene", "tienen", "tenemos", "hacer", "hace",
+        "existe", "algun", "alguna", "empresa", "nuestra", "nuestro", "what", "when", "where", "which", "does",
+        "there", "this", "that", "with", "from", "have", "should", "company",
+    })
+    QUESTION_TOPIC_PREFIX = 5
+
+    def register_question(self, company_short_name: str, user_identifier: str | None, question: str,
+                          channel: str = memory_tools.CHANNEL_CHAT) -> dict:
+        """
+        A question the Memoria could not answer, as the AI or the person says (chat or MCP).
+        Over MCP the answer is written outside IAToolkit, so only they know it went unanswered.
+
+        It enters as a candidate: the hourly review writes it neutrally, gives it a topic,
+        groups it with the open ones and someone is assigned, as with any other. Who asked
+        is not kept. A question whose topic is in content the person may not read (a
+        restricted folder) is not opened for the whole company (BL-015, hallazgo 1).
+        """
+        text = " ".join(str(question or "").split())
+        if not text:
+            return {"status": "error", "error_message": "question is required"}
+        company = self.profile_repo.get_company_by_short_name(company_short_name)
+        if not company:
+            return {"status": "error", "error_message": "company not found"}
+        if self._topic_is_restricted_for(company_short_name, user_identifier, text):
+            return {
+                "status": "success",
+                "registered": False,
+                "reason": "restricted",
+                "message": "No quedó registrada como pregunta abierta de la empresa: ese tema puede estar en "
+                           "información de acceso restringido. Consúltalo con la persona a cargo del área.",
+            }
+        self.question_repo.add(MemoryQuestionCandidate(
+            company_id=company.id,
+            query_text=text[:self.MAX_CANDIDATE_CHARS],
+            source=channel if channel in (memory_tools.CHANNEL_CHAT, memory_tools.CHANNEL_MCP) else memory_tools.CHANNEL_CHAT,
+        ))
+        self.question_repo.commit()
+        return {
+            "status": "success",
+            "registered": True,
+            "message": "Registrada. Dentro de la próxima hora se suma a las preguntas sin respuesta de la empresa "
+                       "(junto a las parecidas) y se le sugiere a quien sabe del tema; cuando alguien la responda, "
+                       "la Memoria lo aprende.",
+        }
+
+    def _topic_is_restricted_for(self, company_short_name: str, user_identifier: str | None, question: str) -> bool:
+        """Whether a page the person may not read covers the question's topic (its title or summary)."""
+        visible = self.memory_service.audiences_for(company_short_name, user_identifier)
+        if visible is None:
+            return False  # they read everything: nothing hidden to give away
+        hidden = [page for page in self.memory_service.memory_wiki_service.list_pages(company_short_name)
+                  if page.get("audience_key") and page.get("audience_key") not in visible]
+        if not hidden:
+            return False
+        words = self._topic_words(question)
+        if not words:
+            return False
+        needed = min(2, len(words))
+        for page in hidden:
+            page_words = self._topic_words(f"{page.get('title') or ''} {page.get('summary') or ''}")
+            if sum(1 for word in words if word in page_words) >= needed:
+                return True
+        return False
+
+    @classmethod
+    def _topic_words(cls, text: str) -> set[str]:
+        """The question's topic words, cut to a prefix so "bono" and "bonos" match."""
+        normalized = MemoryService._normalize_text(text)
+        return {word[:cls.QUESTION_TOPIC_PREFIX] for word in re.findall(r"[a-z0-9]+", normalized)
+                if len(word) >= 4 and word not in cls.QUESTION_STOPWORDS}
+
     def has_pending_work(self, company_short_name: str) -> bool:
         """Whether a scan would find anything new; lets the hourly dispatch skip quiet tenants."""
         company = self.profile_repo.get_company_by_short_name(company_short_name)

{iatoolkit-2.75.4 → iatoolkit-2.76.1}/src/iatoolkit/services/tool_service.py

@@ -121,6 +121,7 @@ class ToolService:
             "iat_memory_save": self._handle_memory_save_tool,
             "iat_memory_pending_questions": self._handle_memory_pending_questions_tool,
             "iat_memory_answer_question": self._handle_memory_answer_question_tool,
+            "iat_memory_register_question": self._handle_memory_register_question_tool,
             "iat_wiki_search": self._handle_wiki_search_tool,
             "iat_wiki_get_page": self._handle_wiki_get_page_tool,
             "iat_web_search": self._handle_web_search_tool,
@@ -429,6 +430,18 @@ class ToolService:
             return {"status": "error", "error_message": "question_id is required"}
         return self._memory_question_service().answer(company_short_name, question_id, str(answer or ""), user_identifier)
 
+    def _handle_memory_register_question_tool(self,
+                                              company_short_name: str,
+                                              question: str,
+                                              user_identifier: str | None = None,
+                                              **kwargs):
+        """Registers a question the Memoria could not answer, so someone who knows answers it."""
+        if not user_identifier:
+            return {"status": "error", "error_message": "memory not available without an identified user"}
+        channel = str(kwargs.get("_iat_runtime_source") or "chat").strip() or "chat"
+        return self._memory_question_service().register_question(
+            company_short_name, user_identifier, str(question or ""), channel=channel)
+
     def _handle_memory_save_tool(self,
                                  company_short_name: str,
                                  content: str,

{iatoolkit-2.75.4 → iatoolkit-2.76.1/src/iatoolkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iatoolkit
-Version: 2.75.4
+Version: 2.76.1
 Summary: IAToolkit
 Author: Fernando Libedinsky
 License-Expression: MIT