lacuscore 1.7.10__tar.gz → 1.8.0__tar.gz

{lacuscore-1.7.10 → lacuscore-1.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lacuscore
-Version: 1.7.10
+Version: 1.8.0
 Summary: Core of Lacus, usable as a module
 Home-page: https://github.com/ail-project/LacusCore
 License: BSD-3-Clause
@@ -26,7 +26,7 @@ Provides-Extra: docs
 Requires-Dist: Sphinx (<7.2) ; (python_version < "3.9") and (extra == "docs")
 Requires-Dist: Sphinx (>=7.2,<8.0) ; (python_version >= "3.9") and (extra == "docs")
 Requires-Dist: defang (>=0.5.3,<0.6.0)
-Requires-Dist: playwrightcapture[recaptcha] (>=1.22.7,<2.0.0)
+Requires-Dist: playwrightcapture[recaptcha] (>=1.23.0,<2.0.0)
 Requires-Dist: redis[hiredis] (>=5.0.1,<6.0.0)
 Requires-Dist: requests (>=2.31.0,<3.0.0)
 Requires-Dist: ua-parser (>=0.18.0,<0.19.0)

{lacuscore-1.7.10 → lacuscore-1.8.0}/lacuscore/lacuscore.py

@@ -27,13 +27,13 @@ from typing import Literal, Any, TypedDict, overload, cast, MutableMapping, Iter
 from uuid import uuid4
 from urllib.parse import urlsplit
 
-from defang import refang  # type: ignore
+from defang import refang  # type: ignore[import-untyped]
 from playwrightcapture import Capture, PlaywrightCaptureException
 from playwrightcapture.capture import CaptureResponse as PlaywrightCaptureResponse
 from redis import Redis
 from redis.exceptions import ConnectionError as RedisConnectionError
 from redis.exceptions import DataError
-from ua_parser import user_agent_parser  # type: ignore
+from ua_parser import user_agent_parser  # type: ignore[import-untyped]
 
 BROWSER = Literal['chromium', 'firefox', 'webkit']
 
@@ -81,7 +81,7 @@ class CaptureResponse(PlaywrightCaptureResponse, TypedDict, total=False):
     '''A capture made by Lacus. With the base64 encoded image and downloaded file decoded to bytes.'''
 
     # Need to make sure the type is what's expected down the line
-    children: list[CaptureResponse] | None  # type: ignore
+    children: list[CaptureResponse] | None  # type: ignore[misc]
 
     status: int
     runtime: float | None
@@ -125,9 +125,9 @@ class CaptureSettings(TypedDict, total=False):
     viewport: dict[str, int] | None
     referer: str | None
     with_favicon: bool
-    force: bool | None
-    recapture_interval: int | None
-    priority: int | None
+    force: bool
+    recapture_interval: int
+    priority: int
     uuid: str | None
 
     depth: int
@@ -268,11 +268,11 @@ class LacusCore():
             if 'url' in settings and settings['url'] is not None:
                 settings['url'] = settings['url'].strip()
             if settings.get('force') is not None:
-                force = settings.pop('force')  # type: ignore
+                force = settings.pop('force', False)
             if settings.get('recapture_interval') is not None:
-                recapture_interval = settings.pop('recapture_interval')  # type: ignore
+                recapture_interval = settings.pop('recapture_interval', 300)
             if settings.get('priority') is not None:
-                priority = settings.pop('priority')  # type: ignore
+                priority = settings.pop('priority', 0)
             to_enqueue = settings
         else:
             to_enqueue = {'depth': depth, 'rendered_hostname_only': rendered_hostname_only}
@@ -331,17 +331,19 @@ class LacusCore():
 
         mapping_capture: dict[str, bytes | float | int | str] = {}
         for key, value in to_enqueue.items():
+            if value is None:
+                continue
             if isinstance(value, bool):
                 mapping_capture[key] = 1 if value else 0
             elif isinstance(value, (list, dict)):
                 if value:
                     mapping_capture[key] = json.dumps(value)
-            elif value is not None and value != '':  # we're ok with 0 for example
-                mapping_capture[key] = value  # type: ignore
+            elif isinstance(value, (bytes, float, int, str)) and value not in ['', b'']:  # we're ok with 0 for example
+                mapping_capture[key] = value
 
         p = self.redis.pipeline()
         p.set(f'lacus:query_hash:{hash_query}', perma_uuid, nx=True, ex=recapture_interval)
-        p.hset(f'lacus:capture_settings:{perma_uuid}', mapping=mapping_capture)  # type: ignore
+        p.hset(f'lacus:capture_settings:{perma_uuid}', mapping=mapping_capture)  # type: ignore[arg-type]
         p.zadd('lacus:to_capture', {perma_uuid: priority if priority is not None else 0})
         try:
             p.execute()
@@ -405,6 +407,10 @@ class LacusCore():
             return CaptureStatus.QUEUED
         elif self.redis.zscore('lacus:ongoing', uuid) is not None:
             return CaptureStatus.ONGOING
+        elif self.redis.exists(f'lacus:capture_settings:{uuid}'):
+            # we might have popped the UUID out of lacus:to_capture
+            # but not pused it in lacus:ongoing yet
+            return CaptureStatus.QUEUED
         elif self.redis.exists(f'lacus:capture_results_hash:{uuid}'):
             return CaptureStatus.DONE
         elif self.redis.exists(f'lacus:capture_results:{uuid}'):
@@ -437,7 +443,7 @@ class LacusCore():
         :param priority: Only for internal use, will decide on the priority of the capture if the try now fails.
         """
         if self.redis.zscore('lacus:ongoing', uuid) is not None:
-            # the capture is ongoing
+            # the capture is already ongoing
             await asyncio.sleep(1)
             return
 
@@ -464,22 +470,22 @@ class LacusCore():
                     if k in ['url', 'document_name', 'browser', 'device_name', 'user_agent',
                              'referer', 'timezone_id', 'locale', 'color_scheme']:
                         # string
-                        to_capture[k] = v.decode()  # type: ignore
+                        to_capture[k] = v.decode()  # type: ignore[literal-required]
                     elif k in ['cookies', 'http_credentials', 'viewport', 'geolocation']:
                         # dicts or list
-                        to_capture[k] = json.loads(v)  # type: ignore
+                        to_capture[k] = json.loads(v)  # type: ignore[literal-required]
                     elif k in ['proxy', 'headers']:
                         # can be dict or str
                         try:
-                            to_capture[k] = json.loads(v)  # type: ignore
+                            to_capture[k] = json.loads(v)  # type: ignore[literal-required]
                         except Exception:
-                            to_capture[k] = v.decode()  # type: ignore
+                            to_capture[k] = v.decode()  # type: ignore[literal-required]
                     elif k in ['general_timeout_in_sec', 'depth']:
                         # int
-                        to_capture[k] = int(v)  # type: ignore
+                        to_capture[k] = int(v)  # type: ignore[literal-required]
                     elif k in ['rendered_hostname_only', 'with_favicon']:
                         # bool
-                        to_capture[k] = bool(int(v))  # type: ignore
+                        to_capture[k] = bool(int(v))  # type: ignore[literal-required]
                     elif k == 'document':
                         document_as_bytes = b64decode(v)
                     else:
@@ -577,15 +583,15 @@ class LacusCore():
                         proxy=proxy,
                         general_timeout_in_sec=general_timeout) as capture:
                     # required by Mypy: https://github.com/python/mypy/issues/3004
-                    capture.headers = to_capture.get('headers')  # type: ignore
-                    capture.cookies = to_capture.get('cookies')  # type: ignore
-                    capture.viewport = to_capture.get('viewport')  # type: ignore
-                    capture.user_agent = to_capture.get('user_agent')  # type: ignore
-                    capture.http_credentials = to_capture.get('http_credentials')  # type: ignore
-                    capture.geolocation = to_capture.get('geolocation')  # type: ignore
-                    capture.timezone_id = to_capture.get('timezone_id')  # type: ignore
-                    capture.locale = to_capture.get('locale')  # type: ignore
-                    capture.color_scheme = to_capture.get('color_scheme')  # type: ignore
+                    capture.headers = to_capture.get('headers')  # type: ignore[assignment]
+                    capture.cookies = to_capture.get('cookies')  # type: ignore[assignment]
+                    capture.viewport = to_capture.get('viewport')  # type: ignore[assignment]
+                    capture.user_agent = to_capture.get('user_agent')  # type: ignore[assignment]
+                    capture.http_credentials = to_capture.get('http_credentials')  # type: ignore[assignment]
+                    capture.geolocation = to_capture.get('geolocation')  # type: ignore[assignment]
+                    capture.timezone_id = to_capture.get('timezone_id')  # type: ignore[assignment]
+                    capture.locale = to_capture.get('locale')  # type: ignore[assignment]
+                    capture.color_scheme = to_capture.get('color_scheme')  # type: ignore[assignment]
                     try:
                         await asyncio.wait_for(capture.initialize_context(), timeout=general_timeout)
                     except (TimeoutError, asyncio.exceptions.TimeoutError):
@@ -748,10 +754,10 @@ class LacusCore():
             if key in ['har', 'cookies', 'potential_favicons', 'html', 'children'] or not results.get(key):
                 continue
             # these entries can be stored directly
-            hash_to_set[key] = results[key]  # type: ignore
+            hash_to_set[key] = results[key]  # type: ignore[literal-required]
 
         if hash_to_set:
-            pipeline.hset(root_key, mapping=hash_to_set)  # type: ignore
+            pipeline.hset(root_key, mapping=hash_to_set)  # type: ignore[arg-type]
             # Make sure the key expires
             pipeline.expire(root_key, 36000)
         else:
@@ -782,19 +788,19 @@ class LacusCore():
                 to_return['children'] = []
                 for child_root_key in sorted(pickle.loads(value)):
                     if child := self._get_capture_response(capture_uuid, child_root_key):
-                        to_return['children'].append(child)  # type: ignore
+                        to_return['children'].append(child)  # type: ignore[union-attr]
             elif key in [b'status']:
                 # The value in an int
-                to_return[key.decode()] = int(value)  # type: ignore
+                to_return[key.decode()] = int(value)  # type: ignore[literal-required]
             elif key in [b'runtime']:
                 # The value is a float
-                to_return[key.decode()] = float(value)  # type: ignore
+                to_return[key.decode()] = float(value)  # type: ignore[literal-required]
             elif key in [b'last_redirected_url', b'error', b'error_name', b'html', b'downloaded_filename']:
                 # the value is a string
-                to_return[key.decode()] = value.decode()  # type: ignore
+                to_return[key.decode()] = value.decode()  # type: ignore[literal-required]
             elif key in [b'png', b'downloaded_file']:
                 # the value is bytes
-                to_return[key.decode()] = value  # type: ignore
+                to_return[key.decode()] = value  # type: ignore[literal-required]
             else:
                 logger.critical(f'Unexpected key in response: {key} - {value}')
         return to_return

{lacuscore-1.7.10 → lacuscore-1.8.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lacuscore"
-version = "1.7.10"
+version = "1.8.0"
 description = "Core of Lacus, usable as a module"
 authors = ["Raphaël Vinot <raphael.vinot@circl.lu>"]
 license = "BSD-3-Clause"
@@ -22,6 +22,7 @@ classifiers = [
     'Programming Language :: Python :: 3.9',
     'Programming Language :: Python :: 3.10',
     'Programming Language :: Python :: 3.11',
+    'Programming Language :: Python :: 3.12',
     'Topic :: Security',
     'Topic :: Internet',
 ]
@@ -33,7 +34,7 @@ Sphinx = [
    {version = "<7.2", python = "<3.9", optional = true},
    {version = "^7.2", python = ">=3.9", optional = true}
 ]
-playwrightcapture = {extras = ["recaptcha"], version = "^1.22.7"}
+playwrightcapture = {extras = ["recaptcha"], version = "^1.23.0"}
 defang = "^0.5.3"
 ua-parser = "^0.18.0"
 redis = {version = "^5.0.1", extras = ["hiredis"]}
@@ -44,14 +45,14 @@ docs = ["Sphinx"]
 [tool.poetry.group.dev.dependencies]
 types-redis = {version = "^4.6.0.20240106"}
 mypy = "^1.8.0"
-types-requests = "^2.31.0.20240106"
+types-requests = "^2.31.0.20240125"
 types-beautifulsoup4 = "^4.12.0.20240106"
 ipython = [
     {version = "<8.13.0", python = "<3.9"},
     {version = "^8.18.0", python = ">=3.9"},
     {version = "^8.19.0", python = ">=3.10"}
 ]
-pytest = "^7.4.4"
+pytest = "^8.0.0"
 
 [build-system]
 requires = ["poetry_core"]