lacuscore 1.14.1__py3-none-any.whl → 1.14.3__py3-none-any.whl

lacuscore/lacuscore.py

@@ -19,7 +19,7 @@ from datetime import date, timedelta
 from ipaddress import ip_address, IPv4Address, IPv6Address
 from tempfile import NamedTemporaryFile
 from typing import Literal, Any, overload, cast, TYPE_CHECKING
-from collections.abc import Iterator
+from collections.abc import AsyncIterator
 from uuid import uuid4
 from urllib.parse import urlsplit
 
@@ -331,20 +331,34 @@ class LacusCore():
         """
         if self.redis.zscore('lacus:to_capture', uuid) is not None:
             return CaptureStatus.QUEUED
-        elif self.redis.zscore('lacus:ongoing', uuid) is not None:
+        if self.redis.zscore('lacus:ongoing', uuid) is not None:
             return CaptureStatus.ONGOING
-        elif self.redis.exists(f'lacus:capture_settings:{uuid}'):
-            # we might have popped the UUID out of lacus:to_capture
-            # but not pused it in lacus:ongoing yet
-            return CaptureStatus.QUEUED
-        elif self.redis.exists(f'lacus:capture_results_hash:{uuid}'):
+        if self.redis.exists(f'lacus:capture_settings:{uuid}'):
+            # we might have a race condition between when the UUID is popped out of lacus:to_capture,
+            # and pushed in lacus:ongoing.
+            # if that's the case, we wait for a sec and check lacus:ongoing again
+            # If it's still not in ongoing, the UUID is broken and can be consdered unknown.
+            # This key is removed anyway once the capture is done.
+            max_checks = 10
+            for i in range(max_checks):
+                time.sleep(.1)
+                if self.redis.zscore('lacus:to_capture', uuid) is not None:
+                    # Could be re-added in that queue if the capture failed, but will be retried
+                    return CaptureStatus.QUEUED
+                if self.redis.zscore('lacus:ongoing', uuid) is not None:
+                    # The capture is actually ongoing now
+                    return CaptureStatus.ONGOING
+            # The UUID is still no anywhere to be found, it's broken.
+            self.redis.delete(f'lacus:capture_settings:{uuid}')
+            return CaptureStatus.UNKNOWN
+        if self.redis.exists(f'lacus:capture_results_hash:{uuid}'):
             return CaptureStatus.DONE
-        elif self.redis.exists(f'lacus:capture_results:{uuid}'):
+        if self.redis.exists(f'lacus:capture_results:{uuid}'):
             # TODO: remove in 1.8.* - old format used last in 1.6, and kept no more than 10H in redis
             return CaptureStatus.DONE
         return CaptureStatus.UNKNOWN
 
-    def consume_queue(self, max_consume: int) -> Iterator[Task]:  # type: ignore[type-arg]
+    async def consume_queue(self, max_consume: int) -> AsyncIterator[Task[None]]:
         """Trigger the capture for captures with the highest priority. Up to max_consume.
 
         :yield: Captures.
@@ -364,6 +378,8 @@ class LacusCore():
             yield task_logger.create_task(self._capture(uuid, priority), name=uuid,
                                           logger=logger,
                                           message='Capture raised an uncaught exception')
+            # Make sur the task starts.
+            await asyncio.sleep(0.5)
 
     async def _capture(self, uuid: str, priority: int) -> None:
         """Trigger a specific capture
@@ -623,9 +639,18 @@ class LacusCore():
             # NOTE: in this block, we absolutely have to make sure the UUID is removed
             #       from the lacus:ongoing sorted set (it is definitely not ongoing anymore)
             #       and optionally re-added to lacus:to_capture if re want to retry it
+            #
+            # In order to have a consistent capture status, the capture UUID must either be in
+            # lacus:ongoing (while ongoing), in lacus:to_capture (on retry), or the result stored (on success).
+            # If the capture fails to be stored in valkey, we must also remove the capture settings
+            # so it is not dangling there.
 
-            if to_capture.document:
-                os.unlink(tmp_f.name)
+            try:
+                if to_capture.document:
+                    os.unlink(tmp_f.name)
+            except UnboundLocalError:
+                # Missing settings, the capture failed.
+                pass
 
             if retry:
                 if self.redis.zcard('lacus:to_capture') == 0:
@@ -655,7 +680,11 @@ class LacusCore():
                         retry_redis_error -= 1
                         await asyncio.sleep(random.randint(5, 10))
                 else:
-                    self.redis.zrem('lacus:ongoing', uuid)
+                    # Unrecoverable redis error, remove the capture settings
+                    p = self.redis.pipeline()
+                    p.delete(f'lacus:capture_settings:{uuid}')
+                    p.zrem('lacus:ongoing', uuid)
+                    p.execute()
                     stats_pipeline.zincrby(f'stats:{today}:errors', 1, 'Redis Connection')
                     logger.critical('Unable to connect to redis and to push the result of the capture.')
 

{lacuscore-1.14.1.dist-info → lacuscore-1.14.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lacuscore
-Version: 1.14.1
+Version: 1.14.3
 Summary: Core of Lacus, usable as a module
 License: BSD-3-Clause
 Author: Raphaël Vinot
@@ -26,7 +26,7 @@ Requires-Dist: async-timeout (>=5.0.1) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3)
 Requires-Dist: dnspython (>=2.7.0)
 Requires-Dist: eval-type-backport (>=0.2.2) ; python_version < "3.10"
-Requires-Dist: playwrightcapture[recaptcha] (>=1.29.2)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.29.3)
 Requires-Dist: pydantic (>=2.11.4)
 Requires-Dist: redis[hiredis] (>=5.3.0,<6.0.0)
 Requires-Dist: requests (>=2.32.3)

{lacuscore-1.14.1.dist-info → lacuscore-1.14.3.dist-info}/RECORD

@@ -1,10 +1,10 @@
 lacuscore/__init__.py,sha256=aLBshQPT9IBDKn5qWrX9A_exqtLFPyLsQiPWdfpAFjA,537
 lacuscore/helpers.py,sha256=d1rMHEA9A9NwfKksa2JvhLu-egaCBf1Vs2F7ALtAhmo,14264
 lacuscore/lacus_monitoring.py,sha256=r6IaYuh6sMq43eOWdZx0fU8p4PWVZlqSD6nr6yOaTUU,2713
-lacuscore/lacuscore.py,sha256=2PixaLmL3LseK4gSpGGV2bDTv742K5h-RLENuFDR8qs,43025
+lacuscore/lacuscore.py,sha256=5JeM2XyMUZJWMIWul1j3GGKmzH8lUZubXkL-ec56rtk,44705
 lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lacuscore/task_logger.py,sha256=2wDotU6r6vn-aKO8nZNdxSuisSj11LlcxuvW60qPL0Y,1909
-lacuscore-1.14.1.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
-lacuscore-1.14.1.dist-info/METADATA,sha256=G_loPnkYOHGy-N00KJVbW1XSitY5k6UEDUUgCZ94eCs,2654
-lacuscore-1.14.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-lacuscore-1.14.1.dist-info/RECORD,,
+lacuscore-1.14.3.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
+lacuscore-1.14.3.dist-info/METADATA,sha256=BGqoy4gu2jancStlS3jBqfyjnVq9Z7APMmSRkKJzZV0,2654
+lacuscore-1.14.3.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+lacuscore-1.14.3.dist-info/RECORD,,