lacuscore 1.14.1__tar.gz → 1.14.3__tar.gz

{lacuscore-1.14.1 → lacuscore-1.14.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lacuscore
-Version: 1.14.1
+Version: 1.14.3
 Summary: Core of Lacus, usable as a module
 License: BSD-3-Clause
 Author: Raphaël Vinot
@@ -26,7 +26,7 @@ Requires-Dist: async-timeout (>=5.0.1) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3)
 Requires-Dist: dnspython (>=2.7.0)
 Requires-Dist: eval-type-backport (>=0.2.2) ; python_version < "3.10"
-Requires-Dist: playwrightcapture[recaptcha] (>=1.29.2)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.29.3)
 Requires-Dist: pydantic (>=2.11.4)
 Requires-Dist: redis[hiredis] (>=5.3.0,<6.0.0)
 Requires-Dist: requests (>=2.32.3)

{lacuscore-1.14.1 → lacuscore-1.14.3}/lacuscore/lacuscore.py

@@ -19,7 +19,7 @@ from datetime import date, timedelta
 from ipaddress import ip_address, IPv4Address, IPv6Address
 from tempfile import NamedTemporaryFile
 from typing import Literal, Any, overload, cast, TYPE_CHECKING
-from collections.abc import Iterator
+from collections.abc import AsyncIterator
 from uuid import uuid4
 from urllib.parse import urlsplit
 
@@ -331,20 +331,34 @@ class LacusCore():
         """
         if self.redis.zscore('lacus:to_capture', uuid) is not None:
             return CaptureStatus.QUEUED
-        elif self.redis.zscore('lacus:ongoing', uuid) is not None:
+        if self.redis.zscore('lacus:ongoing', uuid) is not None:
             return CaptureStatus.ONGOING
-        elif self.redis.exists(f'lacus:capture_settings:{uuid}'):
-            # we might have popped the UUID out of lacus:to_capture
-            # but not pused it in lacus:ongoing yet
-            return CaptureStatus.QUEUED
-        elif self.redis.exists(f'lacus:capture_results_hash:{uuid}'):
+        if self.redis.exists(f'lacus:capture_settings:{uuid}'):
+            # we might have a race condition between when the UUID is popped out of lacus:to_capture,
+            # and pushed in lacus:ongoing.
+            # if that's the case, we wait for a sec and check lacus:ongoing again
+            # If it's still not in ongoing, the UUID is broken and can be consdered unknown.
+            # This key is removed anyway once the capture is done.
+            max_checks = 10
+            for i in range(max_checks):
+                time.sleep(.1)
+                if self.redis.zscore('lacus:to_capture', uuid) is not None:
+                    # Could be re-added in that queue if the capture failed, but will be retried
+                    return CaptureStatus.QUEUED
+                if self.redis.zscore('lacus:ongoing', uuid) is not None:
+                    # The capture is actually ongoing now
+                    return CaptureStatus.ONGOING
+            # The UUID is still no anywhere to be found, it's broken.
+            self.redis.delete(f'lacus:capture_settings:{uuid}')
+            return CaptureStatus.UNKNOWN
+        if self.redis.exists(f'lacus:capture_results_hash:{uuid}'):
             return CaptureStatus.DONE
-        elif self.redis.exists(f'lacus:capture_results:{uuid}'):
+        if self.redis.exists(f'lacus:capture_results:{uuid}'):
             # TODO: remove in 1.8.* - old format used last in 1.6, and kept no more than 10H in redis
             return CaptureStatus.DONE
         return CaptureStatus.UNKNOWN
 
-    def consume_queue(self, max_consume: int) -> Iterator[Task]:  # type: ignore[type-arg]
+    async def consume_queue(self, max_consume: int) -> AsyncIterator[Task[None]]:
         """Trigger the capture for captures with the highest priority. Up to max_consume.
 
         :yield: Captures.
@@ -364,6 +378,8 @@ class LacusCore():
             yield task_logger.create_task(self._capture(uuid, priority), name=uuid,
                                           logger=logger,
                                           message='Capture raised an uncaught exception')
+            # Make sur the task starts.
+            await asyncio.sleep(0.5)
 
     async def _capture(self, uuid: str, priority: int) -> None:
         """Trigger a specific capture
@@ -623,9 +639,18 @@ class LacusCore():
             # NOTE: in this block, we absolutely have to make sure the UUID is removed
             #       from the lacus:ongoing sorted set (it is definitely not ongoing anymore)
             #       and optionally re-added to lacus:to_capture if re want to retry it
+            #
+            # In order to have a consistent capture status, the capture UUID must either be in
+            # lacus:ongoing (while ongoing), in lacus:to_capture (on retry), or the result stored (on success).
+            # If the capture fails to be stored in valkey, we must also remove the capture settings
+            # so it is not dangling there.
 
-            if to_capture.document:
-                os.unlink(tmp_f.name)
+            try:
+                if to_capture.document:
+                    os.unlink(tmp_f.name)
+            except UnboundLocalError:
+                # Missing settings, the capture failed.
+                pass
 
             if retry:
                 if self.redis.zcard('lacus:to_capture') == 0:
@@ -655,7 +680,11 @@ class LacusCore():
                         retry_redis_error -= 1
                         await asyncio.sleep(random.randint(5, 10))
                 else:
-                    self.redis.zrem('lacus:ongoing', uuid)
+                    # Unrecoverable redis error, remove the capture settings
+                    p = self.redis.pipeline()
+                    p.delete(f'lacus:capture_settings:{uuid}')
+                    p.zrem('lacus:ongoing', uuid)
+                    p.execute()
                     stats_pipeline.zincrby(f'stats:{today}:errors', 1, 'Redis Connection')
                     logger.critical('Unable to connect to redis and to push the result of the capture.')
 

{lacuscore-1.14.1 → lacuscore-1.14.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lacuscore"
-version = "1.14.1"
+version = "1.14.3"
 description = "Core of Lacus, usable as a module"
 authors = [
     {name="Raphaël Vinot", email="raphael.vinot@circl.lu"}
@@ -15,7 +15,7 @@ dynamic = [ "classifiers" ]
 
 dependencies = [
     "requests (>=2.32.3)",
-    "playwrightcapture[recaptcha] (>=1.29.2)",
+    "playwrightcapture[recaptcha] (>=1.29.3)",
     "defang (>=0.5.3)",
     "ua-parser[regex] (>=1.0.1)",
     "redis[hiredis] (>=5.3.0,<6.0.0)",
@@ -49,8 +49,8 @@ docs = ["Sphinx (>=8.2.3) ; python_version >= \"3.11\""]
 [tool.poetry.group.dev.dependencies]
 mypy = "^1.15.0"
 types-redis = {version = "^4.6.0.20241004"}
-types-requests = "^2.32.0.20250328"
-types-beautifulsoup4 = "^4.12.0.20250204"
+types-requests = "^2.32.0.20250515"
+types-beautifulsoup4 = "^4.12.0.20250516"
 pytest = "^8.3.5"
 
 [build-system]