lacuscore 1.5.11__tar.gz → 1.5.12__tar.gz

{lacuscore-1.5.11 → lacuscore-1.5.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lacuscore
-Version: 1.5.11
+Version: 1.5.12
 Summary: Core of Lacus, usable as a module
 Home-page: https://github.com/ail-project/LacusCore
 License: BSD-3-Clause
@@ -24,7 +24,7 @@ Classifier: Topic :: Security
 Provides-Extra: docs
 Requires-Dist: Sphinx (>=7.0.1,<8.0.0) ; extra == "docs"
 Requires-Dist: defang (>=0.5.3,<0.6.0)
-Requires-Dist: playwrightcapture[recaptcha] (>=1.20.7,<2.0.0)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.20.8,<2.0.0)
 Requires-Dist: redis[hiredis] (>=4.5.5,<5.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: ua-parser (>=0.16.1,<0.17.0)

{lacuscore-1.5.11 → lacuscore-1.5.12}/lacuscore/lacuscore.py

@@ -17,6 +17,7 @@ import zlib
 
 from asyncio import Task
 from base64 import b64decode, b64encode
+from datetime import date
 from enum import IntEnum, unique
 from logging import LoggerAdapter
 from pathlib import Path
@@ -424,6 +425,7 @@ class LacusCore():
 
         logger = LacusCoreLogAdapter(self.master_logger, {'uuid': uuid})
         self.redis.zadd('lacus:ongoing', {uuid: time.time()})
+        today = date.today().isoformat()
 
         retry = False
         try:
@@ -572,6 +574,12 @@ class LacusCore():
                             rendered_hostname_only=to_capture.get('rendered_hostname_only', True),
                             max_depth_capture_time=self.max_capture_time),
                         timeout=self.max_capture_time)
+                    if 'error' in playwright_result and 'error_name' in playwright_result:
+                        # generate stats
+                        if playwright_result['error_name'] is not None:
+                            self.redis.zincrby(f'stats:{today}:errors', 1, playwright_result['error_name'])
+                            # Expire it in 10 days
+                            self.redis.expire(f'stats:{today}:errors', 3600 * 24 * 10)
             except PlaywrightCaptureException as e:
                 logger.exception(f'Invalid parameters for the capture of {url} - {e}')
                 result = {'error': f'Invalid parameters for the capture of {url} - {e}'}
@@ -607,6 +615,8 @@ class LacusCore():
             else:
                 error_msg = result['error'] if result.get('error') else 'Unknown error'
                 logger.info(f'Retried too many times {url}: {error_msg}')
+                self.redis.sadd(f'stats:{today}:retry_failed', url)
+                self.redis.expire(f'stats:{today}:retry_failed', 3600 * 24 * 10)
 
         except CaptureError:
             if not result:

{lacuscore-1.5.11 → lacuscore-1.5.12}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lacuscore"
-version = "1.5.11"
+version = "1.5.12"
 description = "Core of Lacus, usable as a module"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -32,7 +32,7 @@ include = ['README.md']
 python = "^3.8"
 requests = "^2.31.0"
 Sphinx = { version = "^7.0.1", optional = true }
-playwrightcapture = {extras = ["recaptcha"], version = "^1.20.7"}
+playwrightcapture = {extras = ["recaptcha"], version = "^1.20.8"}
 defang = "^0.5.3"
 ua-parser = "^0.16.1"
 redis = {version = "^4.5.5", extras = ["hiredis"]}
@@ -42,7 +42,7 @@ docs = ["Sphinx"]
 
 [tool.poetry.group.dev.dependencies]
 types-redis = {version = "^4.5.5.2"}
-mypy = "^1.3.0"
+mypy = "^1.4.0"
 types-requests = "^2.31.0.1"
 ipython = [
     {version = "<8.13.0", python = "<3.9"},