iatoolkit 2.76.6__tar.gz → 2.76.8__tar.gz

{iatoolkit-2.76.6/src/iatoolkit.egg-info → iatoolkit-2.76.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iatoolkit
-Version: 2.76.6
+Version: 2.76.8
 Summary: IAToolkit
 Author: Fernando Libedinsky
 License-Expression: MIT

{iatoolkit-2.76.6 → iatoolkit-2.76.8}/src/iatoolkit/__init__.py

@@ -3,7 +3,7 @@
 #
 # IAToolkit is open source software.
 
-__version__ = "2.76.6"
+__version__ = "2.76.8"
 
 # Expose main classes and functions at the top level of the package
 

{iatoolkit-2.76.6 → iatoolkit-2.76.8}/src/iatoolkit/services/company_context_service.py

@@ -446,14 +446,15 @@ class CompanyContextService:
 
         return None
 
-    def get_enriched_database_schema(self, company_short_name: str, db_name: str) -> dict:
+    def get_enriched_database_schema(self, company_short_name: str, db_name: str, *, fresh: bool = False) -> dict:
         """
         Retrieves the physical database structure and enriches it with metadata
-        found in the AssetRepository (YAML files).
+        found in the AssetRepository (YAML files). The structure is cached for a
+        few minutes (SqlService); `fresh` reads it from the database.
         """
         try:
             # 1. Physical Structure (Real Source)
-            structure = self.sql_service.get_database_structure(company_short_name, db_name)
+            structure = self.sql_service.get_database_structure(company_short_name, db_name, fresh=fresh)
 
             # 2. YAML files
             available_files = self.asset_repo.list_files(company_short_name, AssetType.SCHEMA)

{iatoolkit-2.76.6 → iatoolkit-2.76.8}/src/iatoolkit/services/sql_service.py

@@ -22,6 +22,13 @@ class SqlService:
     It maintains a cache of named DatabaseManager instances to avoid reconnecting.
     """
     MAX_QUERY_ROWS = 2000
+    #: How long a database's introspected structure is reused (BL-017). Reading
+    #: it is one query per table per source: `list_sql_sources` went from ~1s to
+    #: 6-13s in production when it began reading every source's tables on each
+    #: call. Shared through Redis by every process; a source saved or deleted,
+    #: or an explicit introspection from the admin, refreshes it at once.
+    STRUCTURE_CACHE_SECONDS = 900
+    STRUCTURE_CACHE_PREFIX = "sql_structure"
     _ALLOWED_QUERY_PREFIXES = ("SELECT", "WITH")
     _BLOCKED_SQL_PATTERN = re.compile(
         r"\b(?:INSERT|UPDATE|DELETE|DROP|ALTER|CREATE|TRUNCATE|MERGE|CALL|COPY|GRANT|REVOKE|"
@@ -104,6 +111,41 @@ class SqlService:
             logging.error(f"Failed to register DB '{db_name}': {e}")
             # We don't raise here to allow other DBs to load if one fails
 
+    @classmethod
+    def _structure_cache_key(cls, company_short_name: str, db_name: str) -> str:
+        return f"{cls.STRUCTURE_CACHE_PREFIX}:{company_short_name}:{db_name}"
+
+    @staticmethod
+    def _read_structure_cache(key: str) -> dict:
+        try:
+            from iatoolkit.infra.redis_session_manager import RedisSessionManager
+            cached = RedisSessionManager.get_json(key, {})
+            return cached if isinstance(cached, dict) else {}
+        except Exception as e:
+            # Without the cache the structure is read from the database, as before.
+            logging.debug(f"SQL structure cache unavailable for '{key}': {e}")
+            return {}
+
+    @classmethod
+    def _write_structure_cache(cls, key: str, structure: dict):
+        if not isinstance(structure, dict) or not structure:
+            return
+        try:
+            from iatoolkit.infra.redis_session_manager import RedisSessionManager
+            RedisSessionManager.set_json(key, structure, ex=cls.STRUCTURE_CACHE_SECONDS)
+        except Exception as e:
+            logging.debug(f"Could not cache the SQL structure '{key}': {e}")
+
+    def invalidate_structure_cache(self, company_short_name: str, db_names: list[str] | None = None):
+        """Forgets the cached structures of these sources (default: every registered one)."""
+        names = set(db_names or []) | set(self.get_db_names(company_short_name) if db_names is None else [])
+        try:
+            from iatoolkit.infra.redis_session_manager import RedisSessionManager
+            for db_name in names:
+                RedisSessionManager.remove(self._structure_cache_key(company_short_name, db_name))
+        except Exception as e:
+            logging.warning(f"Could not invalidate the SQL structure cache of '{company_short_name}': {e}")
+
     def clear_company_connections(self, company_short_name: str):
         keys_to_clear = [key for key in self._db_connections if key[0] == company_short_name]
         for key in keys_to_clear:
@@ -335,14 +377,22 @@ class SqlService:
                 IAToolkitException.ErrorType.DATABASE_ERROR, str(e)
             )
 
-    def get_database_structure(self, company_short_name: str, db_name: str) -> dict:
+    def get_database_structure(self, company_short_name: str, db_name: str, *, fresh: bool = False) -> dict:
         """
-        Introspects the specified database and returns its structure (Tables & Columns).
-        Used for the Schema Editor 2.0
+        The database's structure (tables & columns), introspected at most every
+        STRUCTURE_CACHE_SECONDS. `fresh` reads it from the database and refreshes
+        the cache (the admin's introspection, a schema import).
         """
+        key = self._structure_cache_key(company_short_name, db_name)
+        if not fresh:
+            cached = self._read_structure_cache(key)
+            if cached:
+                return cached
         try:
             provider = self.get_database_provider(company_short_name, db_name)
-            return provider.get_database_structure()
+            structure = provider.get_database_structure()
+            self._write_structure_cache(key, structure)
+            return structure
         except IAToolkitException:
             raise
         except Exception as e:

{iatoolkit-2.76.6 → iatoolkit-2.76.8}/src/iatoolkit/services/sql_source_service.py

@@ -171,7 +171,7 @@ class SqlSourceService:
             source.included_tables = []
 
         persisted = self.sql_source_repo.create_or_update(source)
-        self.refresh_runtime(company_short_name)
+        self._refresh_after_change(company_short_name, [persisted.database])
         return persisted.to_dict()
 
     def update_source(self, company_short_name: str, source_id: int, payload: dict) -> dict:
@@ -237,7 +237,7 @@ class SqlSourceService:
         )
 
         persisted = self.sql_source_repo.create_or_update(source)
-        self.refresh_runtime(company_short_name)
+        self._refresh_after_change(company_short_name, [persisted.database])
         return persisted.to_dict()
 
     def delete_source(self, company_short_name: str, source_id: int) -> None:
@@ -247,7 +247,7 @@ class SqlSourceService:
             raise IAToolkitException(IAToolkitException.ErrorType.NOT_FOUND, "SQL source not found")
 
         self.sql_source_repo.delete(source)
-        self.refresh_runtime(company_short_name)
+        self._refresh_after_change(company_short_name, [source.database])
 
     def sync_from_yaml(self, company_short_name: str, sql_sources: list[dict] | None) -> dict:
         company = self._get_company(company_short_name)
@@ -400,6 +400,14 @@ class SqlSourceService:
         self.sql_service.register_database(company_short_name, normalized_db_name, db_config)
         return normalized_db_name in self.sql_service.get_db_names(company_short_name)
 
+    def _refresh_after_change(self, company_short_name: str, db_names: list[str]):
+        """A source saved or deleted: its cached structure (and every registered one,
+        a rename included) is read again on next use, then the runtime is rebuilt."""
+        registered = self.sql_service.get_db_names(company_short_name)
+        names = list(registered) + [self._normalize_database(name) for name in db_names if name]
+        self.sql_service.invalidate_structure_cache(company_short_name, names)
+        self.refresh_runtime(company_short_name)
+
     def refresh_runtime(self, company_short_name: str) -> dict:
         company = self._get_company(company_short_name)
         self.sql_service.clear_company_connections(company_short_name)

{iatoolkit-2.76.6 → iatoolkit-2.76.8/src/iatoolkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iatoolkit
-Version: 2.76.6
+Version: 2.76.8
 Summary: IAToolkit
 Author: Fernando Libedinsky
 License-Expression: MIT