lacuscore 1.5.11__py3-none-any.whl → 1.5.12__py3-none-any.whl

lacuscore/lacuscore.py

@@ -17,6 +17,7 @@ import zlib
 
 from asyncio import Task
 from base64 import b64decode, b64encode
+from datetime import date
 from enum import IntEnum, unique
 from logging import LoggerAdapter
 from pathlib import Path
@@ -424,6 +425,7 @@ class LacusCore():
 
         logger = LacusCoreLogAdapter(self.master_logger, {'uuid': uuid})
         self.redis.zadd('lacus:ongoing', {uuid: time.time()})
+        today = date.today().isoformat()
 
         retry = False
         try:
@@ -572,6 +574,12 @@ class LacusCore():
                             rendered_hostname_only=to_capture.get('rendered_hostname_only', True),
                             max_depth_capture_time=self.max_capture_time),
                         timeout=self.max_capture_time)
+                    if 'error' in playwright_result and 'error_name' in playwright_result:
+                        # generate stats
+                        if playwright_result['error_name'] is not None:
+                            self.redis.zincrby(f'stats:{today}:errors', 1, playwright_result['error_name'])
+                            # Expire it in 10 days
+                            self.redis.expire(f'stats:{today}:errors', 3600 * 24 * 10)
             except PlaywrightCaptureException as e:
                 logger.exception(f'Invalid parameters for the capture of {url} - {e}')
                 result = {'error': f'Invalid parameters for the capture of {url} - {e}'}
@@ -607,6 +615,8 @@ class LacusCore():
             else:
                 error_msg = result['error'] if result.get('error') else 'Unknown error'
                 logger.info(f'Retried too many times {url}: {error_msg}')
+                self.redis.sadd(f'stats:{today}:retry_failed', url)
+                self.redis.expire(f'stats:{today}:retry_failed', 3600 * 24 * 10)
 
         except CaptureError:
             if not result:

{lacuscore-1.5.11.dist-info → lacuscore-1.5.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lacuscore
-Version: 1.5.11
+Version: 1.5.12
 Summary: Core of Lacus, usable as a module
 Home-page: https://github.com/ail-project/LacusCore
 License: BSD-3-Clause
@@ -24,7 +24,7 @@ Classifier: Topic :: Security
 Provides-Extra: docs
 Requires-Dist: Sphinx (>=7.0.1,<8.0.0) ; extra == "docs"
 Requires-Dist: defang (>=0.5.3,<0.6.0)
-Requires-Dist: playwrightcapture[recaptcha] (>=1.20.7,<2.0.0)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.20.8,<2.0.0)
 Requires-Dist: redis[hiredis] (>=4.5.5,<5.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: ua-parser (>=0.16.1,<0.17.0)

{lacuscore-1.5.11.dist-info → lacuscore-1.5.12.dist-info}/RECORD

@@ -1,9 +1,9 @@
 README.md,sha256=NVr2b3eX2dwOO917TtyGGhCTLtmFbLCFPy9MH8JPUMU,941
 lacuscore/__init__.py,sha256=ytBrQRBXO1Q5yV72qyS16Q7Auqebl3EMhhLQUa0Sg4g,169
 lacuscore/lacus_monitoring.py,sha256=xFf90uXF2yEB_eV-Lh82g9cO1tHkPhTCcSZJ3p6_980,1105
-lacuscore/lacuscore.py,sha256=R5hAn0JmEiydObOJr55KCTYPvsa4USjzW9dzS23-xgY,31022
+lacuscore/lacuscore.py,sha256=JQD_EuGsb49Lz8DnHCS4cRBIqckyJiHfIPpjl5F9Td0,31688
 lacuscore/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lacuscore-1.5.11.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
-lacuscore-1.5.11.dist-info/METADATA,sha256=xjP3OUCuCqBC3HUjYTEi2PdtvqJCqAZAiiahI3XLhK8,2358
-lacuscore-1.5.11.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
-lacuscore-1.5.11.dist-info/RECORD,,
+lacuscore-1.5.12.dist-info/LICENSE,sha256=4C4hLYrIkUD96Ggk-y_Go1Qf7PBZrEm9PSeTGe2nd4s,1516
+lacuscore-1.5.12.dist-info/METADATA,sha256=ImY5JmElpuBoEyPnq8LvuvUTFvbZwBV-5JrGdwZ2Fbk,2358
+lacuscore-1.5.12.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+lacuscore-1.5.12.dist-info/RECORD,,