iatoolkit 2.4.1__tar.gz → 2.5.0__tar.gz

{iatoolkit-2.4.1/src/iatoolkit.egg-info → iatoolkit-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iatoolkit
-Version: 2.4.1
+Version: 2.5.0
 Summary: IAToolkit
 Author: Fernando Libedinsky
 License-Expression: MIT
@@ -12,6 +12,7 @@ Requires-Dist: bcrypt==4.2.1
 Requires-Dist: anthropic>=0.83.0
 Requires-Dist: boto3==1.36.22
 Requires-Dist: botocore==1.36.22
+Requires-Dist: braintrust>=0.17.0
 Requires-Dist: build==1.2.2.post1
 Requires-Dist: click==8.1.8
 Requires-Dist: cryptography==44.0.3

{iatoolkit-2.4.1 → iatoolkit-2.5.0}/pyproject.toml

@@ -29,4 +29,4 @@ version = { attr = "iatoolkit.__version__" }
 dependencies = { file = ["requirements.txt"] }
 
 [tool.setuptools.package-data]
-iatoolkit = ["system_prompts/*.prompt", "templates/**/*", "static/**/*", "locales/*.yaml", "config/*.yaml"]
+iatoolkit = ["templates/**/*", "static/**/*", "locales/*.yaml", "config/*.yaml", "config/system_prompts/*.prompt"]

{iatoolkit-2.4.1 → iatoolkit-2.5.0}/requirements.txt

@@ -2,6 +2,7 @@ bcrypt==4.2.1
 anthropic>=0.83.0
 boto3==1.36.22
 botocore==1.36.22
+braintrust>=0.17.0
 build==1.2.2.post1
 click==8.1.8
 cryptography==44.0.3

{iatoolkit-2.4.1 → iatoolkit-2.5.0}/src/iatoolkit/__init__.py

@@ -3,7 +3,7 @@
 #
 # IAToolkit is open source software.
 
-__version__ = "2.4.1"
+__version__ = "2.5.0"
 
 # Expose main classes and functions at the top level of the package
 

{iatoolkit-2.4.1 → iatoolkit-2.5.0}/src/iatoolkit/core.py

@@ -402,6 +402,7 @@ class IAToolkit:
         from iatoolkit.services.i18n_service import I18nService
         from iatoolkit.services.language_service import LanguageService
         from iatoolkit.services.configuration_service import ConfigurationService
+        from iatoolkit.services.telemetry_service import TelemetryService
         from iatoolkit.services.embedding_service import EmbeddingService
         from iatoolkit.services.history_manager_service import HistoryManagerService
         from iatoolkit.services.tool_service import ToolService
@@ -443,6 +444,7 @@ class IAToolkit:
         binder.bind(I18nService, to=I18nService)
         binder.bind(LanguageService, to=LanguageService)
         binder.bind(ConfigurationService, to=ConfigurationService)
+        binder.bind(TelemetryService, to=TelemetryService)
         binder.bind(EmbeddingService, to=EmbeddingService)
         binder.bind(HistoryManagerService, to=HistoryManagerService)
         binder.bind(ToolService, to=ToolService)

{iatoolkit-2.4.1 → iatoolkit-2.5.0}/src/iatoolkit/locales/en.yaml

@@ -69,6 +69,11 @@ ui:
       primary_input: "Primary Input"
       prompt_args: "Prompt Arguments (Client Data)"
       llm_output: "LLM Output (Raw JSON)"
+      telemetry_label: "Telemetry"
+      telemetry_open_trace: "Open Trace"
+      telemetry_not_recorded: "Not recorded"
+      telemetry_disabled: "Disabled"
+      telemetry_unknown_provider: "unknown provider"
 
   login_widget:
     title: "Sign In"
@@ -1041,6 +1046,8 @@ ui:
     store_response_default: "Provider default"
     store_response_true: "Yes"
     store_response_false: "No"
+    telemetry_enabled_label: "LLM Telemetry"
+    telemetry_enabled_help: "Send this agent's LLM runs to the company telemetry backend when telemetry is enabled in company configuration."
     tools_title: "Tool Selection"
     tools_mode_label: "Tools Mode"
     tools_mode_inherit: "Automatic"

{iatoolkit-2.4.1 → iatoolkit-2.5.0}/src/iatoolkit/locales/es.yaml

@@ -70,6 +70,11 @@ ui:
       primary_input: "Entrada Principal"
       prompt_args: "Argumentos del Prompt (Client Data)"
       llm_output: "Salida LLM (JSON Crudo)"
+      telemetry_label: "Telemetría"
+      telemetry_open_trace: "Abrir traza"
+      telemetry_not_recorded: "No registrada"
+      telemetry_disabled: "Deshabilitada"
+      telemetry_unknown_provider: "provider desconocido"
 
 
   login_widget:
@@ -1044,6 +1049,8 @@ ui:
     store_response_default: "Por defecto del provider"
     store_response_true: "Sí"
     store_response_false: "No"
+    telemetry_enabled_label: "Telemetría LLM"
+    telemetry_enabled_help: "Envía las ejecuciones LLM de este agente al backend de telemetría de la empresa cuando la telemetría esté habilitada en la configuración de la compañía."
     tools_title: "Selección de Tools"
     tools_mode_label: "Modo de Tools"
     tools_mode_inherit: "Automático"

{iatoolkit-2.4.1 → iatoolkit-2.5.0}/src/iatoolkit/services/configuration_service.py

@@ -521,6 +521,56 @@ class ConfigurationService:
                     if raw_x_title is not None and (not isinstance(raw_x_title, str) or not raw_x_title.strip()):
                         add_error("llm.openrouter.x_title", "Must be a non-empty string.")
 
+            telemetry_cfg = config.get("llm", {}).get("telemetry")
+            if telemetry_cfg is not None:
+                if not isinstance(telemetry_cfg, dict):
+                    add_error("llm.telemetry", "Must be a dictionary.")
+                else:
+                    telemetry_enabled = telemetry_cfg.get("enabled", False)
+                    if not isinstance(telemetry_enabled, bool):
+                        add_error("llm.telemetry.enabled", "Must be a boolean.")
+
+                    telemetry_provider = telemetry_cfg.get("provider")
+                    normalized_telemetry_provider = ""
+                    if telemetry_provider is not None:
+                        if not isinstance(telemetry_provider, str) or not telemetry_provider.strip():
+                            add_error("llm.telemetry.provider", "Must be a non-empty string.")
+                        else:
+                            normalized_telemetry_provider = telemetry_provider.strip().lower()
+                            if normalized_telemetry_provider not in {"braintrust"}:
+                                add_error(
+                                    "llm.telemetry.provider",
+                                    "Unsupported provider. Must be 'braintrust'.",
+                                )
+                    elif telemetry_enabled:
+                        add_error("llm.telemetry.provider", "Missing required key: 'provider'.")
+
+                    if normalized_telemetry_provider == "braintrust":
+                        braintrust_cfg = telemetry_cfg.get("braintrust")
+                        if not isinstance(braintrust_cfg, dict):
+                            add_error("llm.telemetry.braintrust", "Must be a dictionary.")
+                        else:
+                            if telemetry_enabled:
+                                project = braintrust_cfg.get("project")
+                                if not isinstance(project, str) or not project.strip():
+                                    add_error("llm.telemetry.braintrust.project", "Must be a non-empty string.")
+
+                                api_key = braintrust_cfg.get("api_key")
+                                if not isinstance(api_key, str) or not api_key.strip():
+                                    add_error("llm.telemetry.braintrust.api_key", "Must be a non-empty string.")
+
+                            api_url = braintrust_cfg.get("api_url")
+                            if api_url is not None:
+                                if not isinstance(api_url, str) or not api_url.strip():
+                                    add_error("llm.telemetry.braintrust.api_url", "Must be a non-empty string.")
+                                else:
+                                    parsed = urlparse(api_url.strip())
+                                    if parsed.scheme.lower() not in {"http", "https"} or not parsed.netloc:
+                                        add_error(
+                                            "llm.telemetry.braintrust.api_url",
+                                            "Must be an absolute HTTP or HTTPS URL.",
+                                        )
+
         # 3. Embedding Provider
         if isinstance(config.get("embedding_provider"), dict):
             if not config.get("embedding_provider", {}).get("provider"):
@@ -678,6 +728,7 @@ class ConfigurationService:
                         "text_verbosity",
                         "prompt_version",
                         "prompt_variant",
+                        "telemetry_enabled",
                     }
                     unknown_option_keys = sorted(
                         key for key in llm_request_options.keys() if key not in allowed_option_keys
@@ -724,6 +775,15 @@ class ConfigurationService:
                                 "Must be a non-empty string.",
                             )
 
+                    if "telemetry_enabled" in llm_request_options and not isinstance(
+                        llm_request_options.get("telemetry_enabled"),
+                        bool,
+                    ):
+                        add_error(
+                            f"prompts[{i}].llm_request_options.telemetry_enabled",
+                            "Must be a boolean.",
+                        )
+
             tool_policy = prompt.get("tool_policy")
             if tool_policy is not None:
                 if not isinstance(tool_policy, dict):
@@ -1110,6 +1170,12 @@ class ConfigurationService:
         provider_cfg = providers_cfg.get(candidate)
         return dict(provider_cfg) if isinstance(provider_cfg, dict) else {}
 
+    def get_llm_telemetry_config(self, company_short_name: str) -> dict:
+        self._ensure_config_loaded(company_short_name)
+        llm_config = self._loaded_configs[company_short_name].get("llm") or {}
+        telemetry_cfg = llm_config.get("telemetry")
+        return dict(telemetry_cfg) if isinstance(telemetry_cfg, dict) else {}
+
 
     def _load_and_merge_configs(self, company_short_name: str) -> dict:
         """

{iatoolkit-2.4.1 → iatoolkit-2.5.0}/src/iatoolkit/services/llm_client_service.py

@@ -23,6 +23,7 @@ from typing import Dict, Optional, List, Any
 from iatoolkit.services.dispatcher_service import Dispatcher
 from iatoolkit.services.storage_service import StorageService
 from iatoolkit.services.structured_output_service import StructuredOutputService
+from iatoolkit.services.telemetry_service import NoopTelemetryService
 
 CONTEXT_ERROR_MESSAGE = 'Tu consulta supera el límite de contexto, utiliza el boton de recarga de contexto.'
 
@@ -36,7 +37,8 @@ class llmClient:
                  llm_proxy: LLMProxy,
                  model_registry: ModelRegistry,
                  storage_service: StorageService,
-                 util: Utility
+                 util: Utility,
+                 telemetry_service=None,
                  ):
         self.llmquery_repo = llmquery_repo
         self.llm_proxy = llm_proxy
@@ -44,6 +46,7 @@ class llmClient:
         self.storage_service = storage_service
         self.util = util
         self._dispatcher = None # Cache for the lazy-loaded dispatcher
+        self._telemetry_service = telemetry_service
 
         # Lazy init to avoid network/bootstrap failures during app startup.
         self.encoding = None
@@ -62,6 +65,18 @@ class llmClient:
             self._dispatcher = current_iatoolkit().get_injector().get(Dispatcher)
         return self._dispatcher
 
+    @property
+    def telemetry_service(self):
+        if self._telemetry_service is None:
+            try:
+                from iatoolkit import current_iatoolkit
+                from iatoolkit.services.telemetry_service import TelemetryService
+
+                self._telemetry_service = current_iatoolkit().get_injector().get(TelemetryService)
+            except Exception:
+                self._telemetry_service = NoopTelemetryService()
+        return self._telemetry_service
+
 
     def invoke(self,
                company: Company,
@@ -81,6 +96,7 @@ class llmClient:
                task_id: Optional[int] = None,
                execution_metadata: Optional[Dict[str, Any]] = None,
                request_metadata: Optional[Dict[str, str]] = None,
+               telemetry_request: Optional[Dict[str, Any]] = None,
                response_contract: Optional[Dict[str, Any]] = None
                ) -> dict:
 
@@ -102,6 +118,7 @@ class llmClient:
         )
         text_payload = request_params["text"]
         reasoning_payload = request_params["reasoning"]
+        telemetry_execution = self.telemetry_service.start_execution(telemetry_request)
 
         try:
             start_time = time.time()
@@ -285,6 +302,22 @@ class llmClient:
                              answer_time=stats['response_time']
                              )
             self.llmquery_repo.add_query(query)
+            telemetry_execution.finalize(
+                query_id=query.id,
+                success=decoded_response.get('status', False),
+                answer_preview=decoded_response.get('answer', ''),
+                metrics={
+                    "total_tokens": combined_stats.get("total_tokens"),
+                    "response_time": combined_stats.get("response_time"),
+                    "sql_retry_count": combined_stats.get("sql_retry_count"),
+                },
+            )
+            telemetry_stats = telemetry_execution.build_stats()
+            if telemetry_stats:
+                query.stats = dict(query.stats or {})
+                query.stats["telemetry"] = telemetry_stats
+                self.llmquery_repo.commit()
+                combined_stats = dict(query.stats or {})
             logging.info(f"finish llm call in {int(time.time() - start_time)} secs..")
             if function_calls:
                 logging.info(f"time within the function calls {f_call_time:.1f} secs.")
@@ -312,9 +345,17 @@ class llmClient:
         except SQLAlchemyError as db_error:
             # rollback
             self.llmquery_repo.session.rollback()
+            telemetry_execution.finalize(
+                success=False,
+                error_message=str(db_error),
+            )
             logging.error(f"Error de base de datos: {str(db_error)}")
             raise db_error
         except OperationalError as e:
+            telemetry_execution.finalize(
+                success=False,
+                error_message=str(e),
+            )
             logging.error(f"Operational error: {str(e)}")
             raise e
         except Exception as e:
@@ -331,6 +372,15 @@ class llmClient:
                              function_calls=f_calls,
                              )
             self.llmquery_repo.add_query(query)
+            telemetry_execution.finalize(
+                query_id=query.id,
+                success=False,
+                error_message=error_message,
+            )
+            telemetry_stats = telemetry_execution.build_stats()
+            if telemetry_stats:
+                query.stats = {"telemetry": telemetry_stats}
+                self.llmquery_repo.commit()
 
             # in case of context error
             if "context_length_exceeded" in str(e):

{iatoolkit-2.4.1 → iatoolkit-2.5.0}/src/iatoolkit/services/prompt_service.py

@@ -345,6 +345,7 @@ class PromptService:
             "text_verbosity",
             "prompt_version",
             "prompt_variant",
+            "telemetry_enabled",
         }
         unknown_keys = sorted(
             key for key in llm_request_options.keys() if key not in allowed_keys
@@ -422,6 +423,15 @@ class PromptService:
                 )
             normalized_options["prompt_variant"] = candidate_prompt_variant
 
+        raw_telemetry_enabled = llm_request_options.get("telemetry_enabled")
+        if raw_telemetry_enabled is not None:
+            if not isinstance(raw_telemetry_enabled, bool):
+                raise IAToolkitException(
+                    IAToolkitException.ErrorType.INVALID_PARAMETER,
+                    "llm_request_options.telemetry_enabled must be a boolean.",
+                )
+            normalized_options["telemetry_enabled"] = raw_telemetry_enabled
+
         return normalized_options
 
     def get_prompts(self, company_short_name: str, include_all: bool = False) -> dict:

{iatoolkit-2.4.1 → iatoolkit-2.5.0}/src/iatoolkit/services/query_service.py

@@ -14,6 +14,7 @@ from iatoolkit.services.history_manager_service import HistoryManagerService
 from iatoolkit.services.context_builder_service import ContextBuilderService
 from iatoolkit.services.attachment_policy_service import AttachmentPolicyService
 from iatoolkit.services.memory_lookup_policy_service import MemoryLookupPolicyService
+from iatoolkit.services.telemetry_service import NoopTelemetryService
 from iatoolkit.common.model_registry import ModelRegistry
 from injector import inject
 import logging
@@ -58,6 +59,7 @@ class QueryService:
                  context_builder: ContextBuilderService,
                  attachment_policy_service: AttachmentPolicyService | None = None,
                  memory_lookup_policy_service: MemoryLookupPolicyService | None = None,
+                 telemetry_service=None,
                  ):
         self.profile_repo = profile_repo
         self.tool_service = tool_service
@@ -71,6 +73,19 @@ class QueryService:
         self.context_builder = context_builder
         self.attachment_policy_service = attachment_policy_service
         self.memory_lookup_policy_service = memory_lookup_policy_service or MemoryLookupPolicyService()
+        self._telemetry_service = telemetry_service
+
+    @property
+    def telemetry_service(self):
+        if self._telemetry_service is None:
+            try:
+                from iatoolkit import current_iatoolkit
+                from iatoolkit.services.telemetry_service import TelemetryService
+
+                self._telemetry_service = current_iatoolkit().get_injector().get(TelemetryService)
+            except Exception:
+                self._telemetry_service = NoopTelemetryService()
+        return self._telemetry_service
 
     @classmethod
     def register_tool_selector_hook(cls, hook: Callable | None):
@@ -294,6 +309,8 @@ class QueryService:
                 prompt_variant = str(raw_llm_request_options.get("prompt_variant") or "").strip()
                 if prompt_variant:
                     normalized_llm_request_options["prompt_variant"] = prompt_variant
+                if isinstance(raw_llm_request_options.get("telemetry_enabled"), bool):
+                    normalized_llm_request_options["telemetry_enabled"] = raw_llm_request_options["telemetry_enabled"]
             contract["llm_request_options"] = normalized_llm_request_options
             raw_resource_bindings = contract.get("resource_bindings")
             normalized_resource_bindings = []
@@ -583,6 +600,29 @@ class QueryService:
             "prompt_variant": prompt_variant[:512],
         }
 
+    def _resolve_telemetry_request(
+        self,
+        *,
+        company_short_name: str,
+        user_identifier: str,
+        effective_model: str,
+        provider: str,
+        task_id: int | None,
+        prompt_output_contract: dict | None,
+        execution_metadata: dict | None = None,
+        request_metadata: dict | None = None,
+    ) -> dict:
+        return self.telemetry_service.resolve_execution_request(
+            company_short_name=company_short_name,
+            prompt_output_contract=prompt_output_contract,
+            model=effective_model,
+            provider=provider,
+            task_id=task_id,
+            user_identifier=user_identifier,
+            execution_metadata=execution_metadata,
+            request_metadata=request_metadata,
+        )
+
     @staticmethod
     def _append_memory_search_instruction(prompt: str, should_suggest_memory_search: bool) -> str:
         if not should_suggest_memory_search:
@@ -825,6 +865,7 @@ class QueryService:
                                       attachment_plan: dict,
                                       execution_metadata: dict,
                                       request_metadata: dict | None,
+                                      telemetry_request: dict | None,
                                       prompt_output_contract: dict,
                                       history_handle: HistoryHandle,
                                       ignore_history: bool) -> tuple[dict, HistoryHandle]:
@@ -850,6 +891,7 @@ class QueryService:
                 store=store,
                 execution_metadata=execution_metadata,
                 request_metadata=request_metadata,
+                telemetry_request=telemetry_request,
                 response_contract=prompt_output_contract if prompt_output_contract.get("schema") else None,
             )
             return response, history_handle
@@ -907,6 +949,7 @@ class QueryService:
                 store=store,
                 execution_metadata=execution_metadata,
                 request_metadata=request_metadata,
+                telemetry_request=telemetry_request,
                 response_contract=prompt_output_contract if prompt_output_contract.get("schema") else None,
             )
             return response, retry_history_handle
@@ -1340,6 +1383,16 @@ class QueryService:
                 "provider": provider,
                 "company_defaults": self._resolve_company_attachment_defaults(company_short_name, prompt_name=prompt_name),
             }
+            telemetry_request = self._resolve_telemetry_request(
+                company_short_name=company_short_name,
+                user_identifier=user_identifier,
+                effective_model=effective_model,
+                provider=provider,
+                task_id=task_id,
+                prompt_output_contract=prompt_output_contract,
+                execution_metadata=execution_metadata,
+                request_metadata=request_metadata,
+            )
 
             response, history_handle = self._invoke_with_history_recovery(
                 company=company,
@@ -1358,6 +1411,7 @@ class QueryService:
                 attachment_plan=attachment_plan,
                 execution_metadata=execution_metadata,
                 request_metadata=request_metadata,
+                telemetry_request=telemetry_request,
                 prompt_output_contract=prompt_output_contract,
                 history_handle=history_handle,
                 ignore_history=ignore_history,

{iatoolkit-2.4.1 → iatoolkit-2.5.0}/src/iatoolkit/services/system_prompt_catalog.py

@@ -10,7 +10,7 @@ import yaml
 
 SYSTEM_PROMPTS_CONFIG_PACKAGE = "iatoolkit.config"
 SYSTEM_PROMPTS_CONFIG_FILENAME = "system_prompts_pack.yaml"
-SYSTEM_PROMPTS_PACKAGE = "iatoolkit.system_prompts"
+SYSTEM_PROMPTS_DIRNAME = "system_prompts"
 SYSTEM_PROMPT_SECTION_ORDER = (
     "identity",
     "business_context",
@@ -209,7 +209,11 @@ def _load_catalog() -> tuple[dict, ...]:
 
 @lru_cache(maxsize=64)
 def _read_prompt_text(filename: str) -> str:
-    return importlib.resources.read_text(SYSTEM_PROMPTS_PACKAGE, filename)
+    return (
+        importlib.resources.files(SYSTEM_PROMPTS_CONFIG_PACKAGE)
+        .joinpath(SYSTEM_PROMPTS_DIRNAME, filename)
+        .read_text(encoding="utf-8")
+    )
 
 
 def get_system_prompt_entries() -> list[dict]: