lacuscore 1.14.0__tar.gz → 1.14.2__tar.gz

{lacuscore-1.14.0 → lacuscore-1.14.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lacuscore
-Version: 1.14.0
+Version: 1.14.2
 Summary: Core of Lacus, usable as a module
 License: BSD-3-Clause
 Author: Raphaël Vinot
@@ -26,9 +26,9 @@ Requires-Dist: async-timeout (>=5.0.1) ; python_version < "3.11"
 Requires-Dist: defang (>=0.5.3)
 Requires-Dist: dnspython (>=2.7.0)
 Requires-Dist: eval-type-backport (>=0.2.2) ; python_version < "3.10"
-Requires-Dist: playwrightcapture[recaptcha] (>=1.29.0)
-Requires-Dist: pydantic (>=2.11.3)
-Requires-Dist: redis[hiredis] (>=5.2.1)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.29.3)
+Requires-Dist: pydantic (>=2.11.4)
+Requires-Dist: redis[hiredis] (>=5.3.0,<6.0.0)
 Requires-Dist: requests (>=2.32.3)
 Requires-Dist: typing-extensions (>=4.12.2,<5.0.0) ; python_version < "3.12"
 Requires-Dist: ua-parser[regex] (>=1.0.1)

{lacuscore-1.14.0 → lacuscore-1.14.2}/lacuscore/lacuscore.py

@@ -269,6 +269,7 @@ class LacusCore():
         p = self.redis.pipeline()
         p.set(f'lacus:query_hash:{hash_query}', perma_uuid, nx=True, ex=recapture_interval)
         p.hset(f'lacus:capture_settings:{perma_uuid}', mapping=to_enqueue.redis_dump())
+        p.expire(f'lacus:capture_settings:{perma_uuid}', self.max_capture_time * 2)
         p.zadd('lacus:to_capture', {perma_uuid: priority if priority is not None else 0})
         try:
             p.execute()
@@ -330,15 +331,29 @@ class LacusCore():
         """
         if self.redis.zscore('lacus:to_capture', uuid) is not None:
             return CaptureStatus.QUEUED
-        elif self.redis.zscore('lacus:ongoing', uuid) is not None:
+        if self.redis.zscore('lacus:ongoing', uuid) is not None:
             return CaptureStatus.ONGOING
-        elif self.redis.exists(f'lacus:capture_settings:{uuid}'):
-            # we might have popped the UUID out of lacus:to_capture
-            # but not pused it in lacus:ongoing yet
-            return CaptureStatus.QUEUED
-        elif self.redis.exists(f'lacus:capture_results_hash:{uuid}'):
+        if self.redis.exists(f'lacus:capture_settings:{uuid}'):
+            # we might have a race condition between when the UUID is popped out of lacus:to_capture,
+            # and pushed in lacus:ongoing.
+            # if that's the case, we wait for a sec and check lacus:ongoing again
+            # If it's still not in ongoing, the UUID is broken and can be consdered unknown.
+            # This key is removed anyway once the capture is done.
+            max_checks = 10
+            for i in range(max_checks):
+                time.sleep(.1)
+                if self.redis.zscore('lacus:to_capture', uuid) is not None:
+                    # Could be re-added in that queue if the capture failed, but will be retried
+                    return CaptureStatus.QUEUED
+                if self.redis.zscore('lacus:ongoing', uuid) is not None:
+                    # The capture is actually ongoing now
+                    return CaptureStatus.ONGOING
+            # The UUID is still no anywhere to be found, it's broken.
+            self.redis.delete(f'lacus:capture_settings:{uuid}')
+            return CaptureStatus.UNKNOWN
+        if self.redis.exists(f'lacus:capture_results_hash:{uuid}'):
             return CaptureStatus.DONE
-        elif self.redis.exists(f'lacus:capture_results:{uuid}'):
+        if self.redis.exists(f'lacus:capture_results:{uuid}'):
             # TODO: remove in 1.8.* - old format used last in 1.6, and kept no more than 10H in redis
             return CaptureStatus.DONE
         return CaptureStatus.UNKNOWN
@@ -622,6 +637,11 @@ class LacusCore():
             # NOTE: in this block, we absolutely have to make sure the UUID is removed
             #       from the lacus:ongoing sorted set (it is definitely not ongoing anymore)
             #       and optionally re-added to lacus:to_capture if re want to retry it
+            #
+            # In order to have a consistent capture status, the capture UUID must either be in
+            # lacus:ongoing (while ongoing), in lacus:to_capture (on retry), or the result stored (on success).
+            # If the capture fails to be stored in valkey, we must also remove the capture settings
+            # so it is not dangling there.
 
             if to_capture.document:
                 os.unlink(tmp_f.name)
@@ -654,7 +674,11 @@ class LacusCore():
                         retry_redis_error -= 1
                         await asyncio.sleep(random.randint(5, 10))
                 else:
-                    self.redis.zrem('lacus:ongoing', uuid)
+                    # Unrecoverable redis error, remove the capture settings
+                    p = self.redis.pipeline()
+                    p.delete(f'lacus:capture_settings:{uuid}')
+                    p.zrem('lacus:ongoing', uuid)
+                    p.execute()
                     stats_pipeline.zincrby(f'stats:{today}:errors', 1, 'Redis Connection')
                     logger.critical('Unable to connect to redis and to push the result of the capture.')
 

{lacuscore-1.14.0 → lacuscore-1.14.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lacuscore"
-version = "1.14.0"
+version = "1.14.2"
 description = "Core of Lacus, usable as a module"
 authors = [
     {name="Raphaël Vinot", email="raphael.vinot@circl.lu"}
@@ -15,13 +15,13 @@ dynamic = [ "classifiers" ]
 
 dependencies = [
     "requests (>=2.32.3)",
-    "playwrightcapture[recaptcha] (>=1.29.0)",
+    "playwrightcapture[recaptcha] (>=1.29.3)",
     "defang (>=0.5.3)",
     "ua-parser[regex] (>=1.0.1)",
-    "redis [hiredis] (>=5.2.1)",
+    "redis[hiredis] (>=5.3.0,<6.0.0)",
     "dnspython (>=2.7.0)",
     "async-timeout (>=5.0.1) ; python_version < \"3.11\"",
-    "pydantic (>=2.11.3)",
+    "pydantic (>=2.11.4)",
     "eval-type-backport (>=0.2.2) ; python_version < \"3.10\"",
     "typing-extensions (>=4.12.2,<5.0.0) ; python_version < \"3.12\""
 ]
@@ -49,8 +49,8 @@ docs = ["Sphinx (>=8.2.3) ; python_version >= \"3.11\""]
 [tool.poetry.group.dev.dependencies]
 mypy = "^1.15.0"
 types-redis = {version = "^4.6.0.20241004"}
-types-requests = "^2.32.0.20250328"
-types-beautifulsoup4 = "^4.12.0.20250204"
+types-requests = "^2.32.0.20250515"
+types-beautifulsoup4 = "^4.12.0.20250516"
 pytest = "^8.3.5"
 
 [build-system]