iatoolkit 2.27.4__tar.gz → 2.27.5__tar.gz

{iatoolkit-2.27.4/src/iatoolkit.egg-info → iatoolkit-2.27.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iatoolkit
-Version: 2.27.4
+Version: 2.27.5
 Summary: IAToolkit
 Author: Fernando Libedinsky
 License-Expression: MIT

{iatoolkit-2.27.4 → iatoolkit-2.27.5}/src/iatoolkit/__init__.py

@@ -3,7 +3,7 @@
 #
 # IAToolkit is open source software.
 
-__version__ = "2.27.4"
+__version__ = "2.27.5"
 
 # Expose main classes and functions at the top level of the package
 

{iatoolkit-2.27.4 → iatoolkit-2.27.5}/src/iatoolkit/infra/connectors/google_cloud_storage_connector.py

@@ -47,7 +47,12 @@ class GoogleCloudStorageConnector(FileConnector):
             {
                 "path": blob.name,  # Nombre o "ruta" del blob en el bucket
                 "name": blob.name.split("/")[-1],  # Nombre del archivo (última parte del path)
-                "metadata": {"size": blob.size}  # Incluye tamaño como metadata (u otros metadatos relevantes)
+                "metadata": {
+                    "size": blob.size,
+                    "last_modified": blob.updated.isoformat() if blob.updated else None,
+                    "md5_hash": blob.md5_hash,
+                    "crc32c": blob.crc32c,
+                }
             }
             for blob in blobs
         ]

{iatoolkit-2.27.4 → iatoolkit-2.27.5}/src/iatoolkit/repositories/models.py

@@ -664,6 +664,7 @@ class KnowledgeWikiSyncRun(Base):
     pages_seen = Column(Integer, nullable=False, default=0)
     pages_indexed = Column(Integer, nullable=False, default=0)
     pages_failed = Column(Integer, nullable=False, default=0)
+    pages_skipped = Column(Integer, nullable=False, default=0)
     errors = Column(JSON_NATIVE, nullable=False, default=list)
     metadata_json = Column(JSON_NATIVE, nullable=False, default=dict)
     started_at = Column(DateTime, default=datetime.now, nullable=False)

{iatoolkit-2.27.4 → iatoolkit-2.27.5}/src/iatoolkit/services/tenant_wiki_service.py

@@ -194,6 +194,7 @@ class TenantWikiService:
         seen_paths: set[str] = set()
         indexed = 0
         failed = 0
+        skipped = 0
 
         try:
             files = self.storage_service.list_files(
@@ -207,13 +208,22 @@ class TenantWikiService:
             ]
             for item in source_files:
                 storage_key = str(item.get("path") or "").strip()
+                current_metadata = item.get("metadata") if isinstance(item.get("metadata"), dict) else {}
+                relative_path = self._relative_path(wiki.root_storage_key, storage_key)
+                existing_page = self.knowledge_wiki_repo.get_page_by_path(wiki.id, relative_path)
+                if existing_page and self._file_metadata_unchanged(existing_page, current_metadata):
+                    seen_paths.add(relative_path)
+                    existing_page.last_synced_at = sync_started_at
+                    self.knowledge_wiki_repo.save_page(existing_page)
+                    skipped += 1
+                    continue
                 try:
                     markdown = self.markdown_wiki_service.read_markdown(company_short_name, storage_key)
                     page_payload = self._parse_markdown_page(
                         root_storage_key=wiki.root_storage_key,
                         storage_key=storage_key,
                         markdown=markdown,
-                        file_metadata=item.get("metadata") if isinstance(item.get("metadata"), dict) else {},
+                        file_metadata=current_metadata,
                     )
                     seen_paths.add(page_payload["path"])
                     unique_slug = self._resolve_unique_slug(wiki.id, page_payload["slug"], page_payload["path"])
@@ -248,6 +258,7 @@ class TenantWikiService:
             run.pages_seen = len(source_files)
             run.pages_indexed = indexed
             run.pages_failed = failed
+            run.pages_skipped = skipped
             run.errors = errors
             run.metadata_json = {**(run.metadata_json or {}), "archived_pages": archived}
             run.finished_at = datetime.now()
@@ -1381,6 +1392,29 @@ class TenantWikiService:
             return False
         return path != self.INDEX_FILENAME
 
+    @staticmethod
+    def _file_metadata_unchanged(page: KnowledgeWikiPage, current_metadata: dict) -> bool:
+        if not isinstance(current_metadata, dict) or not current_metadata:
+            return False
+        source_meta = page.source_meta if isinstance(page.source_meta, dict) else {}
+        stored_file_meta = source_meta.get("file_metadata") if isinstance(source_meta.get("file_metadata"), dict) else {}
+        if not stored_file_meta:
+            return False
+        for key in ("md5_hash", "crc32c"):
+            current_val = str(current_metadata.get(key) or "").strip()
+            stored_val = str(stored_file_meta.get(key) or "").strip()
+            if current_val and stored_val:
+                return current_val == stored_val
+        current_size = current_metadata.get("size")
+        stored_size = stored_file_meta.get("size")
+        current_lm = str(current_metadata.get("last_modified") or "").strip()
+        stored_lm = str(stored_file_meta.get("last_modified") or "").strip()
+        if current_size is not None and stored_size is not None and current_lm and stored_lm:
+            return str(current_size) == str(stored_size) and current_lm == stored_lm
+        if current_size is not None and stored_size is not None:
+            return str(current_size) == str(stored_size)
+        return False
+
     @staticmethod
     def _normalize_tags(value: Any) -> list[str]:
         if isinstance(value, str):
@@ -1519,6 +1553,7 @@ class TenantWikiService:
             "pages_seen": run.pages_seen,
             "pages_indexed": run.pages_indexed,
             "pages_failed": run.pages_failed,
+            "pages_skipped": run.pages_skipped,
             "errors": run.errors or [],
             "metadata": run.metadata_json or {},
             "started_at": run.started_at.isoformat() if run.started_at else None,

{iatoolkit-2.27.4 → iatoolkit-2.27.5/src/iatoolkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: iatoolkit
-Version: 2.27.4
+Version: 2.27.5
 Summary: IAToolkit
 Author: Fernando Libedinsky
 License-Expression: MIT