iatoolkit 2.77.1__tar.gz → 2.77.3__tar.gz

{iatoolkit-2.77.1/src/iatoolkit.egg-info → iatoolkit-2.77.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iatoolkit
-Version: 2.77.1
+Version: 2.77.3
 Summary: IAToolkit
 Author: Fernando Libedinsky
 License-Expression: MIT

{iatoolkit-2.77.1 → iatoolkit-2.77.3}/src/iatoolkit/__init__.py

@@ -3,7 +3,7 @@
 #
 # IAToolkit is open source software.
 
-__version__ = "2.77.1"
+__version__ = "2.77.3"
 
 # Expose main classes and functions at the top level of the package
 

{iatoolkit-2.77.1 → iatoolkit-2.77.3}/src/iatoolkit/infra/openrouter_decision_client.py

@@ -66,12 +66,15 @@ class OpenRouterDecisionClient:
         read_timeout_seconds: float = 60.0,
         retry_attempts: int = DEFAULT_RETRY_ATTEMPTS,
         user: str | None = None,
+        assets: dict[str, dict[str, str]] | None = None,
     ) -> DecisionResult:
         payload: dict[str, Any] = {
             "model": model,
             "state": state,
             "questions": {name: question.to_wire() for name, question in questions.items()},
         }
+        if assets:
+            payload["assets"] = assets
         if provider_policy:
             payload["provider"] = dict(provider_policy)
         if user:
@@ -109,7 +112,12 @@ class OpenRouterDecisionClient:
                 return DecisionResult.from_response(body, requested_model=model, latency_ms=latency_ms)
 
             message = self._error_message(response)
-            if response.status_code in RETRYABLE_STATUS and attempt < attempts:
+            # With images, a 5xx is how an oversized request answers (measured: a
+            # bare 500); sending it again only repeats it. Rate limits still retry.
+            retryable = response.status_code in RETRYABLE_STATUS and (
+                not assets or response.status_code == 429
+            )
+            if retryable and attempt < attempts:
                 logging.warning(
                     "Decision model '%s' answered %s (attempt %s/%s): %s",
                     model, response.status_code, attempt, attempts, message,

{iatoolkit-2.77.1 → iatoolkit-2.77.3}/src/iatoolkit/repositories/llm_query_repo.py

@@ -14,6 +14,7 @@ from iatoolkit.repositories.models import (
 from injector import inject
 from iatoolkit.repositories.database_manager import DatabaseManager
 from sqlalchemy import or_, and_
+from sqlalchemy.orm.attributes import flag_modified
 from iatoolkit.repositories.models import CONTEXT_INIT_REQUEST_SOURCE
 from typing import List
 
@@ -233,6 +234,9 @@ class LLMQueryRepo:
                 or dict(getattr(prompt, "runtime_policy", None) or {})
             )
             prompt.decision_spec = getattr(new_prompt, "decision_spec", None)
+            # The order of the questions is part of the agent, and dicts compare
+            # equal regardless of it: without this a reorder is never written.
+            flag_modified(prompt, "decision_spec")
         else:
             if new_prompt.active is None:
                 new_prompt.active = True

{iatoolkit-2.77.1 → iatoolkit-2.77.3}/src/iatoolkit/services/configuration_service.py

@@ -658,6 +658,12 @@ class ConfigurationService:
                 else:
                     if decisions_cfg.get("provider") is not None and not isinstance(decisions_cfg.get("provider"), dict):
                         add_error("llm.decisions.provider", "Must be a dictionary (e.g. {zdr: true}).")
+                    raw_max_images = decisions_cfg.get("max_images")
+                    if raw_max_images is not None and (
+                        isinstance(raw_max_images, bool) or not isinstance(raw_max_images, int)
+                        or not 1 <= raw_max_images <= 20
+                    ):
+                        add_error("llm.decisions.max_images", "Must be an integer between 1 and 20.")
                     raw_path = decisions_cfg.get("path")
                     if raw_path is not None and (not isinstance(raw_path, str) or not raw_path.strip().startswith("/")):
                         add_error("llm.decisions.path", "Must be a path starting with '/'.")

{iatoolkit-2.77.1 → iatoolkit-2.77.3}/src/iatoolkit/services/context_builder_service.py

@@ -577,6 +577,22 @@ class ContextBuilderService:
             images=[],
         ) or "").strip()
 
+    def prepare_decision_attachments(self, company: Company, files: list, *, mode: str | None,
+                                     max_images: int):
+        """A decision agent's attachments (BL-008, Etapa 2): images to send and text for the case."""
+        from iatoolkit.services.decision_attachments import prepare_decision_attachments
+
+        company_short_name = str(getattr(company, "short_name", "") or "default")
+        return prepare_decision_attachments(
+            files,
+            mode=mode,
+            max_images=max_images,
+            decode=self.util.normalize_base64_payload,
+            extract_text=lambda filename, content: self.parsing_service.extract_text_for_context(
+                filename=filename, content=content, company_short_name=company_short_name,
+            ),
+        )
+
     def build_user_turn_prompt(self,
                                company: Company,
                                user_identifier: str,

iatoolkit-2.77.3/src/iatoolkit/services/decision_attachments.py

@@ -0,0 +1,248 @@
+# Copyright (c) 2024 Fernando Libedinsky
+# Product: IAToolkit
+#
+# IAToolkit is open source software.
+
+"""Attachments of a decision agent (BL-008, Etapa 2): images and PDFs.
+
+Decision models take images only as PNG or JPEG `assets`, and each one limits
+what it accepts: Jev refuses anything over ~1568 px on its longest side, and
+every model tested answered a bare 500 to a 4 MB image. A PDF is not an asset
+at all. So nothing goes out as it came in:
+
+- every image, and every PDF page sent as an image, is normalized to a JPEG of
+  at most 1568 px, which fits every model, keeps the request small, lowers the
+  cost (image tokens grow with resolution) and accepts any format Pillow reads;
+- a PDF's text, when the agent asks for it, joins the case as text, through
+  the same extractor the chat agents use.
+
+What reaches the model is decided by the agent's attachment mode, the same
+setting chat agents have:
+
+| mode                    | image         | PDF                         |
+|-------------------------|---------------|-----------------------------|
+| `extracted_only`        | refused       | its text joins the case     |
+| `native_only`           | sent as image | its pages are sent as images|
+| `native_plus_extracted` | sent as image | text and pages              |
+"""
+
+from __future__ import annotations
+
+import hashlib
+import io
+from dataclasses import dataclass, field
+from typing import Any, Callable
+
+from iatoolkit.common.exceptions import IAToolkitException
+
+MODE_EXTRACTED = "extracted_only"
+MODE_NATIVE = "native_only"
+MODE_BOTH = "native_plus_extracted"
+MODES = (MODE_EXTRACTED, MODE_NATIVE, MODE_BOTH)
+
+#: Longest side of every image sent. Jev's limit, measured (1568 passes, 2048
+#: fails); every other model tested accepts it.
+MAX_IMAGE_SIDE = 1568
+JPEG_QUALITY = 85
+#: Bytes of one image after normalization. 2.7 MB passed and 4 MB failed with a
+#: 500; a 1568 px JPEG is far below this, so it only guards odd inputs.
+MAX_IMAGE_BYTES = 2_500_000
+DEFAULT_MAX_IMAGES = 5
+
+_IMAGE_EXTENSIONS = (".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".tif", ".tiff", ".heic", ".heif")
+
+
+@dataclass(frozen=True)
+class DecisionImage:
+    """One image as sent: a normalized JPEG, named after where it came from."""
+    name: str
+    data: bytes
+    source: str
+    page: int | None = None
+
+    @property
+    def sha256(self) -> str:
+        return hashlib.sha256(self.data).hexdigest()
+
+
+@dataclass
+class PreparedAttachments:
+    images: list[DecisionImage] = field(default_factory=list)
+    #: Extracted text, ready to follow the rendered case.
+    text: str = ""
+    #: What was received and what was made of it, for the record (never the bytes).
+    manifest: list[dict[str, Any]] = field(default_factory=list)
+
+
+def _refuse(message: str) -> IAToolkitException:
+    return IAToolkitException(IAToolkitException.ErrorType.INVALID_PARAMETER, message)
+
+
+def normalize_image(data: bytes, name: str = "image") -> bytes:
+    """Any image Pillow reads, as a JPEG of at most MAX_IMAGE_SIDE px."""
+    from PIL import Image, ImageOps
+
+    try:
+        image = Image.open(io.BytesIO(data))
+        image.load()
+    except Exception as exc:
+        raise _refuse(f"The attachment '{name}' is not an image this platform can read ({exc}).") from exc
+
+    image = ImageOps.exif_transpose(image)
+    if image.mode in ("RGBA", "LA", "P"):
+        # JPEG has no transparency: flatten on white, as the page it came from.
+        rgba = image.convert("RGBA")
+        background = Image.new("RGB", rgba.size, "white")
+        background.paste(rgba, mask=rgba.split()[-1])
+        image = background
+    elif image.mode != "RGB":
+        image = image.convert("RGB")
+    image.thumbnail((MAX_IMAGE_SIDE, MAX_IMAGE_SIDE))
+
+    quality = JPEG_QUALITY
+    while True:
+        buffer = io.BytesIO()
+        image.save(buffer, "JPEG", quality=quality, optimize=True)
+        encoded = buffer.getvalue()
+        if len(encoded) <= MAX_IMAGE_BYTES or quality <= 40:
+            break
+        quality -= 15
+    if len(encoded) > MAX_IMAGE_BYTES:
+        raise _refuse(f"The image '{name}' is still over {MAX_IMAGE_BYTES // 1_000_000} MB after resizing.")
+    return encoded
+
+
+def render_pdf_pages(data: bytes, name: str) -> list[bytes]:
+    """Every page of a PDF as a normalized JPEG."""
+    import fitz
+
+    try:
+        document = fitz.open(stream=data, filetype="pdf")
+    except Exception as exc:
+        raise _refuse(f"The PDF '{name}' could not be opened ({exc}).") from exc
+    pages = []
+    with document:
+        for page in document:
+            longest = max(page.rect.width, page.rect.height) or 1
+            zoom = min(MAX_IMAGE_SIDE / longest, 4.0)
+            pixmap = page.get_pixmap(matrix=fitz.Matrix(zoom, zoom))
+            pages.append(normalize_image(pixmap.tobytes("png"), name))
+    return pages
+
+
+def pdf_page_count(data: bytes) -> int:
+    import fitz
+
+    with fitz.open(stream=data, filetype="pdf") as document:
+        return document.page_count
+
+
+def _is_image(filename: str) -> bool:
+    return filename.lower().endswith(_IMAGE_EXTENSIONS)
+
+
+def _is_pdf(filename: str) -> bool:
+    return filename.lower().endswith(".pdf")
+
+
+def prepare_decision_attachments(
+    files: list[dict] | None,
+    *,
+    mode: str | None,
+    max_images: int = DEFAULT_MAX_IMAGES,
+    decode: Callable[[Any], bytes],
+    extract_text: Callable[[str, bytes], str],
+) -> PreparedAttachments:
+    """What a decision agent sends for its attachments, or an error that says why not.
+
+    `decode` turns a file's payload (base64, possibly a data URL) into bytes;
+    `extract_text` is the extractor chat agents use for documents.
+    """
+    prepared = PreparedAttachments()
+    if not files:
+        return prepared
+    mode = str(mode or MODE_EXTRACTED).strip().lower()
+    if mode not in MODES:
+        mode = MODE_EXTRACTED
+    native = mode in (MODE_NATIVE, MODE_BOTH)
+    extracted = mode in (MODE_EXTRACTED, MODE_BOTH)
+    text_parts: list[str] = []
+
+    for document in files:
+        filename = str(document.get("file_id") or document.get("filename") or document.get("name") or "").strip()
+        payload = document.get("base64") or document.get("content")
+        if not filename or not payload:
+            raise _refuse("An attachment has no name or no content.")
+        data = decode(payload)
+        if isinstance(data, str):
+            data = data.encode("utf-8")
+        entry: dict[str, Any] = {
+            "name": filename,
+            "bytes": len(data),
+            "sha256": hashlib.sha256(data).hexdigest(),
+        }
+
+        if _is_image(filename):
+            if not native:
+                raise _refuse(
+                    f"'{filename}' is an image, and this agent's attachment mode only sends extracted text. "
+                    "Change it to native to send images."
+                )
+            _check_room(prepared, 1, max_images, filename)
+            prepared.images.append(DecisionImage(
+                name=f"img_{len(prepared.images) + 1}",
+                data=normalize_image(data, filename),
+                source=filename,
+            ))
+            entry["sent_as"] = "image"
+            entry["images"] = 1
+
+        elif _is_pdf(filename):
+            entry["pages"] = pdf_page_count(data)
+            sent_as = []
+            if extracted:
+                text = (extract_text(filename, data) or "").strip()
+                if text:
+                    text_parts.append(f"Documento adjunto '{filename}':\n{text}")
+                    sent_as.append("text")
+                elif not native:
+                    raise _refuse(
+                        f"The PDF '{filename}' has no text to extract (it looks scanned). "
+                        "Change the agent's attachment mode to native to send its pages as images."
+                    )
+            if native:
+                _check_room(prepared, entry["pages"], max_images, filename)
+                for page_number, page in enumerate(render_pdf_pages(data, filename), start=1):
+                    prepared.images.append(DecisionImage(
+                        name=f"img_{len(prepared.images) + 1}",
+                        data=page,
+                        source=filename,
+                        page=page_number,
+                    ))
+                sent_as.append("pages")
+                entry["images"] = entry["pages"]
+            entry["sent_as"] = "+".join(sent_as)
+
+        else:
+            if not extracted:
+                raise _refuse(
+                    f"'{filename}' is neither an image nor a PDF; in native mode a decision agent sends only those."
+                )
+            text = (extract_text(filename, data) or "").strip()
+            if not text:
+                raise _refuse(f"No text could be extracted from '{filename}'.")
+            text_parts.append(f"Documento adjunto '{filename}':\n{text}")
+            entry["sent_as"] = "text"
+
+        prepared.manifest.append(entry)
+
+    prepared.text = "\n\n".join(text_parts)
+    return prepared
+
+
+def _check_room(prepared: PreparedAttachments, needed: int, max_images: int, filename: str) -> None:
+    if len(prepared.images) + needed > max_images:
+        raise _refuse(
+            f"'{filename}' would take the images sent to {len(prepared.images) + needed}; "
+            f"a decision agent sends at most {max_images} (llm.decisions.max_images)."
+        )

{iatoolkit-2.77.1 → iatoolkit-2.77.3}/src/iatoolkit/services/decision_service.py

@@ -12,6 +12,7 @@ endpoint but none of its adapters.
 
 from __future__ import annotations
 
+import base64
 import json
 import logging
 from typing import Any
@@ -25,6 +26,7 @@ from iatoolkit.common.decision_types import (
     parse_decision_spec,
 )
 from iatoolkit.common.exceptions import IAToolkitException
+from iatoolkit.services.decision_attachments import DecisionImage
 from iatoolkit.infra.llm_proxy import LLMProxy
 from iatoolkit.infra.openrouter_decision_client import OpenRouterDecisionClient
 from iatoolkit.services.configuration_service import ConfigurationService
@@ -98,6 +100,7 @@ class DecisionService:
         decision_spec: dict,
         user_identifier: str | None = None,
         telemetry_execution: Any = None,
+        images: list[DecisionImage] | None = None,
     ) -> DecisionResult:
         try:
             questions = parse_decision_spec(decision_spec)
@@ -116,6 +119,10 @@ class DecisionService:
                 "The case to decide on is empty: the agent template rendered no text.",
             )
         self._warn_if_too_long(company_short_name, model_key, model_config, state_text)
+        images = list(images or [])
+        if images:
+            self._check_takes_images(model_key, model_config)
+        wire_state, assets = self.build_state(model_key, state_text, images)
 
         client_config = self.llm_proxy.resolve_client_config(
             company_short_name, DECISION_PROVIDER, model_config=model_config
@@ -131,7 +138,11 @@ class DecisionService:
                 name="iatoolkit.decision",
                 span_type="llm",
                 event={
-                    "input": {"state": state_text, "questions": {n: q.to_wire() for n, q in questions.items()}},
+                    "input": {
+                        "state": state_text,
+                        "questions": {n: q.to_wire() for n, q in questions.items()},
+                        "images": [{"name": i.name, "source": i.source, "page": i.page} for i in images],
+                    },
                     "metadata": {"model": wire_model, "company": company_short_name},
                 },
             )
@@ -143,8 +154,9 @@ class DecisionService:
                 api_key=client_config.get("api_key") or "",
                 url=url,
                 model=wire_model,
-                state=state_text,
+                state=wire_state,
                 questions=questions,
+                assets=assets,
                 provider_policy=provider_policy if isinstance(provider_policy, dict) else None,
                 default_headers=client_config.get("default_headers") or {},
                 connect_timeout_seconds=client_config.get("connect_timeout_seconds") or 10.0,
@@ -176,6 +188,45 @@ class DecisionService:
         )
         return result
 
+    @staticmethod
+    def _check_takes_images(model: str, model_config: dict) -> None:
+        capabilities = model_config.get("capabilities") if isinstance(model_config.get("capabilities"), dict) else {}
+        modalities = {str(item).strip().lower() for item in capabilities.get("input_modalities") or []}
+        if "image" not in modalities:
+            raise IAToolkitException(
+                IAToolkitException.ErrorType.INVALID_PARAMETER,
+                f"Decision model '{model}' does not read images; choose one that does, "
+                "or set the agent's attachment mode to extracted text.",
+            )
+
+    @staticmethod
+    def build_state(model: str, state_text: str, images: list[DecisionImage]) -> tuple[Any, dict | None]:
+        """The `state` and `assets` for a case with images (measured on 9 models).
+
+        Every model that reads images takes them as whole top-level items of an
+        array state, referenced by their asset key: `[case, "img_1", ...]`. Only
+        Jev also reads a reference inside text or a nested object. ScaleDown
+        takes no assets at all and reads one image as a document instead.
+        """
+        if not images:
+            return state_text, None
+        if model.startswith("scaledown/"):
+            if len(images) > 1:
+                raise IAToolkitException(
+                    IAToolkitException.ErrorType.INVALID_PARAMETER,
+                    f"Decision model '{model}' reads one image per case; {len(images)} were sent.",
+                )
+            return {
+                "text": state_text,
+                "document": base64.b64encode(images[0].data).decode("ascii"),
+                "document_mime_type": "image/jpeg",
+            }, None
+        assets = {
+            image.name: {"data_base64": base64.b64encode(image.data).decode("ascii"), "format": "jpg"}
+            for image in images
+        }
+        return [state_text, *[image.name for image in images]], assets
+
     @staticmethod
     def _warn_if_too_long(company_short_name: str, model: str, model_config: dict, state_text: str) -> None:
         try:

{iatoolkit-2.77.1 → iatoolkit-2.77.3}/src/iatoolkit/services/llm_client_service.py

@@ -146,7 +146,9 @@ class llmClient:
                         task_id: Optional[int] = None,
                         execution_metadata: Optional[Dict[str, Any]] = None,
                         telemetry_request: Optional[Dict[str, Any]] = None,
-                        execution_trace: Optional[Dict[str, Any]] = None) -> dict:
+                        execution_trace: Optional[Dict[str, Any]] = None,
+                        images: Optional[list] = None,
+                        attachments: Optional[List[Dict[str, Any]]] = None) -> dict:
         """Runs a decision agent's model (BL-008) and records it like any agent execution.
 
         Same table, same `stats` keys as a chat turn — `model`, tokens, response
@@ -159,6 +161,12 @@ class llmClient:
         telemetry_execution = self.telemetry_service.start_execution(telemetry_request)
         telemetry_execution.record_input({"state": state, "questions": (decision_spec or {}).get("questions")})
         base_stats: dict = {"model": model, "kind": "decision"}
+        if attachments:
+            # What was attached and how it was sent; never the bytes.
+            base_stats["attachments"] = {
+                "count": len(attachments),
+                "images_sent": len(images or []),
+            }
         for key, value in (execution_metadata or {}).items():
             if value is not None:
                 base_stats[key] = value
@@ -171,6 +179,7 @@ class llmClient:
                 decision_spec=decision_spec,
                 user_identifier=user_identifier,
                 telemetry_execution=telemetry_execution,
+                images=images,
             )
         except Exception as e:
             error_message = getattr(e, "message", None) or str(e)
@@ -226,6 +235,7 @@ class llmClient:
                              "decision": result.to_dict(),
                              "state": state,
                              "decision_spec": decision_spec,
+                             "attachments": list(attachments or []),
                          },
                          function_calls=[],
                          stats=stats,

{iatoolkit-2.77.1 → iatoolkit-2.77.3}/src/iatoolkit/services/query_service.py

@@ -1387,9 +1387,6 @@ class QueryService:
                             prompt_output_contract: dict,
                             execution_trace: dict | None) -> dict:
         company_short_name = company.short_name
-        if files:
-            return {"error": True,
-                    "error_message": "Decision agents do not take attachments yet: send the case in the agent's fields."}
         if not str(model or "").strip() and not str(prompt_output_contract.get("llm_model") or "").strip():
             return {"error": True,
                     "error_message": f"Agent '{prompt_name}' has no decision model: choose one in its settings."}
@@ -1409,6 +1406,17 @@ class QueryService:
         )
         question = json.dumps({'prompt': prompt_name, 'data': safe_client_data}, default=str)
 
+        # Attachments follow the agent's attachment mode: extracted text joins the
+        # case, images (and PDF pages, in native mode) travel as images.
+        prepared = self.context_builder.prepare_decision_attachments(
+            company,
+            files or [],
+            mode=prompt_output_contract.get("attachment_mode"),
+            max_images=self._decision_max_images(company_short_name),
+        )
+        if prepared.text:
+            state = f"{state}\n\n{prepared.text}" if state else prepared.text
+
         execution_metadata: dict = {}
         request_source = str(safe_client_data.get("source") or "").strip().lower() or None
         if request_source:
@@ -1435,8 +1443,21 @@ class QueryService:
             execution_metadata=execution_metadata,
             telemetry_request=telemetry_request,
             execution_trace=execution_trace,
+            images=prepared.images,
+            attachments=prepared.manifest,
         )
 
+    def _decision_max_images(self, company_short_name: str) -> int:
+        from iatoolkit.services.decision_attachments import DEFAULT_MAX_IMAGES
+
+        llm_config = self.configuration_service.get_configuration(company_short_name, "llm") or {}
+        decisions = llm_config.get("decisions") if isinstance(llm_config, dict) else None
+        raw = decisions.get("max_images") if isinstance(decisions, dict) else None
+        try:
+            return int(raw) if raw is not None else DEFAULT_MAX_IMAGES
+        except (TypeError, ValueError):
+            return DEFAULT_MAX_IMAGES
+
     def llm_query(self,
                   company_short_name: str,
                   user_identifier: str,

{iatoolkit-2.77.1 → iatoolkit-2.77.3/src/iatoolkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iatoolkit
-Version: 2.77.1
+Version: 2.77.3
 Summary: IAToolkit
 Author: Fernando Libedinsky
 License-Expression: MIT

{iatoolkit-2.77.1 → iatoolkit-2.77.3}/src/iatoolkit.egg-info/SOURCES.txt

@@ -130,6 +130,7 @@ src/iatoolkit/services/branding_service.py
 src/iatoolkit/services/company_context_service.py
 src/iatoolkit/services/configuration_service.py
 src/iatoolkit/services/context_builder_service.py
+src/iatoolkit/services/decision_attachments.py
 src/iatoolkit/services/decision_service.py
 src/iatoolkit/services/dispatcher_service.py
 src/iatoolkit/services/embedding_service.py