lacuscore 1.14.0__py3-none-any.whl → 1.14.2__py3-none-any.whl

lacuscore/lacuscore.py

@@ -269,6 +269,7 @@ class LacusCore():
         p = self.redis.pipeline()
         p.set(f'lacus:query_hash:{hash_query}', perma_uuid, nx=True, ex=recapture_interval)
         p.hset(f'lacus:capture_settings:{perma_uuid}', mapping=to_enqueue.redis_dump())
+        p.expire(f'lacus:capture_settings:{perma_uuid}', self.max_capture_time * 2)
         p.zadd('lacus:to_capture', {perma_uuid: priority if priority is not None else 0})
         try:
             p.execute()
@@ -330,15 +331,29 @@ class LacusCore():
         """
         if self.redis.zscore('lacus:to_capture', uuid) is not None:
             return CaptureStatus.QUEUED
-        elif self.redis.zscore('lacus:ongoing', uuid) is not None:
+        if self.redis.zscore('lacus:ongoing', uuid) is not None:
             return CaptureStatus.ONGOING
-        elif self.redis.exists(f'lacus:capture_settings:{uuid}'):
-            # we might have popped the UUID out of lacus:to_capture
-            # but not pused it in lacus:ongoing yet
-            return CaptureStatus.QUEUED
-        elif self.redis.exists(f'lacus:capture_results_hash:{uuid}'):
+        if self.redis.exists(f'lacus:capture_settings:{uuid}'):
+            # we might have a race condition between when the UUID is popped out of lacus:to_capture,
+            # and pushed in lacus:ongoing.
+            # if that's the case, we wait for a sec and check lacus:ongoing again
+            # If it's still not in ongoing, the UUID is broken and can be consdered unknown.
+            # This key is removed anyway once the capture is done.
+            max_checks = 10
+            for i in range(max_checks):
+                time.sleep(.1)
+                if self.redis.zscore('lacus:to_capture', uuid) is not None:
+                    # Could be re-added in that queue if the capture failed, but will be retried
+                    return CaptureStatus.QUEUED
+                if self.redis.zscore('lacus:ongoing', uuid) is not None:
+                    # The capture is actually ongoing now
+                    return CaptureStatus.ONGOING
+            # The UUID is still no anywhere to be found, it's broken.
+            self.redis.delete(f'lacus:capture_settings:{uuid}')
+            return CaptureStatus.UNKNOWN
+        if self.redis.exists(f'lacus:capture_results_hash:{uuid}'):
             return CaptureStatus.DONE
-        elif self.redis.exists(f'lacus:capture_results:{uuid}'):
+        if self.redis.exists(f'lacus:capture_results:{uuid}'):
             # TODO: remove in 1.8.* - old format used last in 1.6, and kept no more than 10H in redis
             return CaptureStatus.DONE
         return CaptureStatus.UNKNOWN
@@ -622,6 +637,11 @@ class LacusCore():
             # NOTE: in this block, we absolutely have to make sure the UUID is removed
             #       from the lacus:ongoing sorted set (it is definitely not ongoing anymore)
             #       and optionally re-added to lacus:to_capture if re want to retry it
+            #
+            # In order to have a consistent capture status, the capture UUID must either be in
+            # lacus:ongoing (while ongoing), in lacus:to_capture (on retry), or the result stored (on success).
+            # If the capture fails to be stored in valkey, we must also remove the capture settings
+            # so it is not dangling there.
 
             if to_capture.document:
                 os.unlink(tmp_f.name)
@@ -654,7 +674,11 @@ class LacusCore():
                         retry_redis_error -= 1
                         await asyncio.sleep(random.randint(5, 10))
                 else:
-                    self.redis.zrem('lacus:ongoing', uuid)
+                    # Unrecoverable redis error, remove the capture settings
+                    p = self.redis.pipeline()
+                    p.delete(f'lacus:capture_settings:{uuid}')
+                    p.zrem('lacus:ongoing', uuid)
+                    p.execute()
                     stats_pipeline.zincrby(f'stats:{today}:errors', 1, 'Redis Connection')
                     logger.critical('Unable to connect to redis and to push the result of the capture.')
 

{lacuscore-1.14.0.dist-info → lacuscore-1.14.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lacuscore
-Version: 1.14.0
+Version: 1.14.2
 Summary: Core of Lacus, usable as a module
 License: BSD-3-Clause
 Author: Raphaël Vinot
@@ -26,9 +26,9 @@ Requires-Dist: async-timeout (>=5.0.1) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3)
 Requires-Dist: dnspython (>=2.7.0)
 Requires-Dist: eval-type-backport (>=0.2.2) ; python_version < "3.10"
-Requires-Dist: playwrightcapture[recaptcha] (>=1.29.0)
-Requires-Dist: pydantic (>=2.11.3)
-Requires-Dist: redis[hiredis] (>=5.2.1)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.29.3)
+Requires-Dist: pydantic (>=2.11.4)
+Requires-Dist: redis[hiredis] (>=5.3.0,<6.0.0)
 Requires-Dist: requests (>=2.32.3)
 Requires-Dist: typing-extensions (>=4.12.2,<5.0.0) ; python_version < "3.12"
 Requires-Dist: ua-parser[regex] (>=1.0.1)

{lacuscore-1.14.0.dist-info → lacuscore-1.14.2.dist-info}/RECORD

@@ -1,10 +1,10 @@
 lacuscore/__init__.py,sha256=aLBshQPT9IBDKn5qWrX9A_exqtLFPyLsQiPWdfpAFjA,537
 lacuscore/helpers.py,sha256=d1rMHEA9A9NwfKksa2JvhLu-egaCBf1Vs2F7ALtAhmo,14264
 lacuscore/lacus_monitoring.py,sha256=r6IaYuh6sMq43eOWdZx0fU8p4PWVZlqSD6nr6yOaTUU,2713
-lacuscore/lacuscore.py,sha256=cENEza8A5NjW6aO9jImRYZTa5RKlMTHkRITAtMMYTnc,42941
+lacuscore/lacuscore.py,sha256=zUkfSSn9_tFE9vnFo309obHJdvBBfhVjwe7IDL7csoU,44492
 lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lacuscore/task_logger.py,sha256=2wDotU6r6vn-aKO8nZNdxSuisSj11LlcxuvW60qPL0Y,1909
-lacuscore-1.14.0.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
-lacuscore-1.14.0.dist-info/METADATA,sha256=-9LdOO7lsKp5UFHkS7JfC4Jg2O3cBjopjqnp1E6x1Zw,2647
-lacuscore-1.14.0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-lacuscore-1.14.0.dist-info/RECORD,,
+lacuscore-1.14.2.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
+lacuscore-1.14.2.dist-info/METADATA,sha256=qkvp08WeFCrDYogV4NcoFFvGGlgvWZC3P6-RwkgtBJ0,2654
+lacuscore-1.14.2.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+lacuscore-1.14.2.dist-info/RECORD,,

{lacuscore-1.14.0.dist-info → lacuscore-1.14.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.2
+Generator: poetry-core 2.1.3
 Root-Is-Purelib: true
 Tag: py3-none-any