iatoolkit 2.60.0__tar.gz → 2.60.2__tar.gz

{iatoolkit-2.60.0/src/iatoolkit.egg-info → iatoolkit-2.60.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iatoolkit
-Version: 2.60.0
+Version: 2.60.2
 Summary: IAToolkit
 Author: Fernando Libedinsky
 License-Expression: MIT

{iatoolkit-2.60.0 → iatoolkit-2.60.2}/src/iatoolkit/__init__.py

@@ -3,7 +3,7 @@
 #
 # IAToolkit is open source software.
 
-__version__ = "2.60.0"
+__version__ = "2.60.2"
 
 # Expose main classes and functions at the top level of the package
 

{iatoolkit-2.60.0 → iatoolkit-2.60.2}/src/iatoolkit/infra/connectors/google_drive_connector.py

@@ -104,6 +104,9 @@ class DriveChangesPage:
 
 class GoogleDriveConnector(FileConnector):
     PAGE_SIZE = 1000
+    # Reads retry Google's transient errors (5xx, 429) with exponential backoff,
+    # as googleapiclient does with num_retries; writes are not retried.
+    NUM_RETRIES = 3
 
     def __init__(
         self,
@@ -193,7 +196,7 @@ class GoogleDriveConnector(FileConnector):
                 fileId=file_id,
                 fields=FILE_FIELDS,
                 supportsAllDrives=True,
-            ).execute()
+            ).execute(num_retries=self.NUM_RETRIES)
         except HttpError as exc:
             if getattr(exc, "status_code", None) in (403, 404) or getattr(getattr(exc, "resp", None), "status", None) in (403, 404):
                 return None
@@ -211,7 +214,7 @@ class GoogleDriveConnector(FileConnector):
                 pageToken=page_token,
                 supportsAllDrives=True,
                 includeItemsFromAllDrives=True,
-            ).execute()
+            ).execute(num_retries=self.NUM_RETRIES)
             for payload in response.get("files", []):
                 yield DriveFile.from_api(payload)
             page_token = response.get("nextPageToken")
@@ -269,14 +272,14 @@ class GoogleDriveConnector(FileConnector):
         downloader = MediaIoBaseDownload(buffer, request)
         done = False
         while not done:
-            _, done = downloader.next_chunk()
+            _, done = downloader.next_chunk(num_retries=GoogleDriveConnector.NUM_RETRIES)
         return buffer.getvalue()
 
     # --- Changes ------------------------------------------------------------
 
     def get_start_page_token(self) -> str:
         """Where to start reading changes from: everything after now."""
-        response = self.drive_service.changes().getStartPageToken(supportsAllDrives=True).execute()
+        response = self.drive_service.changes().getStartPageToken(supportsAllDrives=True).execute(num_retries=self.NUM_RETRIES)
         return str(response.get("startPageToken") or "")
 
     def list_changes(self, page_token: str) -> DriveChangesPage:
@@ -292,7 +295,7 @@ class GoogleDriveConnector(FileConnector):
             supportsAllDrives=True,
             includeItemsFromAllDrives=True,
             spaces="drive",
-        ).execute()
+        ).execute(num_retries=self.NUM_RETRIES)
         changes = [
             DriveChange(
                 file_id=str(change.get("fileId") or ""),

{iatoolkit-2.60.0 → iatoolkit-2.60.2}/src/iatoolkit/repositories/memory_question_repo.py

@@ -6,7 +6,7 @@
 from __future__ import annotations
 
 from injector import inject
-from sqlalchemy import desc
+from sqlalchemy import desc, func
 
 from iatoolkit.repositories.database_manager import DatabaseManager
 from iatoolkit.repositories.models import (
@@ -88,6 +88,15 @@ class MemoryQuestionRepo:
             .all()
         )
 
+    def last_query_id_before(self, company_id: int, moment) -> int:
+        """The newest iat_queries.id of the company created before `moment` (0 when none)."""
+        value = (
+            self.session.query(func.max(LLMQuery.id))
+            .filter(LLMQuery.company_id == company_id, LLMQuery.created_at < moment)
+            .scalar()
+        )
+        return int(value or 0)
+
     def has_queries_after(self, company_id: int, last_query_id: int) -> bool:
         return (
             self.session.query(LLMQuery.id)

{iatoolkit-2.60.0 → iatoolkit-2.60.2}/src/iatoolkit/services/memory_compiler_service.py

@@ -62,12 +62,8 @@ class MemoryCompilerService:
         "type": "object",
         "properties": {
             "action": {"type": "string", "enum": ["create", "update", "skip"]},
-            "target_page_id": {
-                "anyOf": [
-                    {"type": "integer"},
-                    {"type": "null"},
-                ]
-            },
+            # Nullable as a type list: the structured-output validator rejects null under anyOf.
+            "target_page_id": {"type": ["integer", "null"]},
             "title": {"type": "string"},
             "summary": {"type": "string"},
             "markdown": {"type": "string"},

{iatoolkit-2.60.0 → iatoolkit-2.60.2}/src/iatoolkit/services/memory_question_service.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 import json
 import logging
 from contextlib import contextmanager
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from injector import inject
 from sqlalchemy import text
@@ -47,6 +47,12 @@ class MemoryQuestionService:
     MAX_QUESTION_CHARS = 1000
     MAX_ANSWER_CHARS = 1500
     MAX_CANDIDATE_CHARS = 500
+    # A company's first review (or one whose cursor never advanced) starts with
+    # the last days, not with the whole history.
+    FIRST_SCAN_DAYS = 7
+    REF_KEYS = ("ref", "id", "turn", "turn_ref", "turn_id")
+    TRUE_WORDS = {"true", "yes", "si", "sí", "1"}
+    FALSE_WORDS = {"false", "no", "0"}
     # First key of the two-key Postgres advisory lock; the second is the company id.
     ADVISORY_LOCK_NAMESPACE = 734002
 
@@ -62,8 +68,9 @@ class MemoryQuestionService:
                         "answered": {"type": "boolean"},
                         "question": {"type": "string"},
                         "topic": {"type": "string"},
-                        "same_as_question_id": {"anyOf": [{"type": "integer"}, {"type": "null"}]},
-                        "same_as_ref": {"anyOf": [{"type": "string"}, {"type": "null"}]},
+                        # Nullable as a type list: core's validator rejects null under anyOf.
+                        "same_as_question_id": {"type": ["integer", "null"]},
+                        "same_as_ref": {"type": ["string", "null"]},
                     },
                     "required": ["ref", "answered"],
                     "additionalProperties": False,
@@ -201,6 +208,10 @@ class MemoryQuestionService:
     def _scan(self, company: Company) -> dict:
         counts = {"turns": 0, "reviewed": 0, "answered": 0, "new_questions": 0, "repeated": 0, "batches": 0}
         cursor = self.question_repo.get_cursor(company.id)
+        if not cursor.last_query_id:
+            cursor.last_query_id = self.question_repo.last_query_id_before(
+                company.id, datetime.now() - timedelta(days=self.FIRST_SCAN_DAYS))
+            self.question_repo.commit()
         queries = self.question_repo.list_queries_after(company.id, cursor.last_query_id, self.MAX_QUERIES_PER_SCAN)
         candidates = self.question_repo.list_pending_candidates(company.id, self.MAX_QUERIES_PER_SCAN)
         counts["turns"] = len(queries)
@@ -338,13 +349,60 @@ class MemoryQuestionService:
         except Exception as exc:
             logging.warning("Memory question scan for %s could not reach the model: %s", company.short_name, exc)
             return None
-        structured = response.get("structured_output") or response.get("parsed_output")
-        if isinstance(structured, list):
-            structured = {"items": structured}
-        items = structured.get("items") if isinstance(structured, dict) else None
-        if not isinstance(items, list):
+        items = self._extract_items(response.get("structured_output"))
+        if items is None:
+            items = self._extract_items(response.get("parsed_output"))
+        if items is None:
+            # A top-level list never reaches parsed_output (it carries objects only).
+            items = self._extract_items(self._json_from_text(response.get("answer")))
+        if items is None:
+            logging.warning(
+                "Memory question scan for %s did not understand the model's answer: %s",
+                company.short_name, str(response.get("answer") or "")[:500],
+            )
+            return None
+        return [normalized for normalized in (self._normalize_item(item) for item in items) if normalized]
+
+    @staticmethod
+    def _extract_items(value) -> list | None:
+        """The list of classified turns, whether it comes bare or under some key."""
+        if isinstance(value, list):
+            return [item for item in value if isinstance(item, dict)]
+        if not isinstance(value, dict):
+            return None
+        if isinstance(value.get("items"), list):
+            return [item for item in value["items"] if isinstance(item, dict)]
+        for nested in value.values():
+            if isinstance(nested, list) and all(isinstance(item, dict) for item in nested):
+                return list(nested)
+        for key in ("answer", "additional_data", "result", "data"):
+            if isinstance(value.get(key), (dict, list)):
+                return MemoryQuestionService._extract_items(value[key])
+        return None
+
+    @staticmethod
+    def _json_from_text(text) -> object | None:
+        raw = str(text or "").strip()
+        if raw.startswith("```"):
+            raw = raw.strip("`")
+            raw = raw[4:] if raw.lower().startswith("json") else raw
+        try:
+            return json.loads(raw.strip())
+        except (TypeError, ValueError):
+            return None
+
+    @classmethod
+    def _normalize_item(cls, item: dict) -> dict | None:
+        ref = next((item.get(key) for key in cls.REF_KEYS if item.get(key) not in (None, "")), None)
+        if ref is None:
+            return None
+        answered = item.get("answered")
+        if isinstance(answered, str):
+            word = answered.strip().lower()
+            answered = True if word in cls.TRUE_WORDS else False if word in cls.FALSE_WORDS else None
+        if not isinstance(answered, bool):
             return None
-        return [item for item in items if isinstance(item, dict)]
+        return {**item, "ref": str(ref), "answered": answered}
 
     def _apply(self, company: Company, batch: list[dict], items: list[dict], open_questions: list[MemoryQuestion],
                counts: dict) -> None:

{iatoolkit-2.60.0 → iatoolkit-2.60.2/src/iatoolkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iatoolkit
-Version: 2.60.0
+Version: 2.60.2
 Summary: IAToolkit
 Author: Fernando Libedinsky
 License-Expression: MIT