polyswarm-engine 3.2.0__py2.py3-none-any.whl → 3.2.2__py2.py3-none-any.whl

polyswarm_engine/__init__.py

@@ -1,5 +1,5 @@
 # flake8: noqa
-__VERSION__ = '3.2.0'
+__VERSION__ = '3.2.2'
 
 from .bidutils import (
     bid_max,

polyswarm_engine/backend.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 import contextlib
 import logging
 import typing as t
@@ -9,7 +10,12 @@ import celery
 
 from polyswarm_engine import exceptions
 from polyswarm_engine.bounty import get_bounty_expiration, get_bounty_tasked_at, CANNOT_FETCH
-from polyswarm_engine.settings import PSENGINE_METADATA_ARCHTECTURE, PSENGINE_METADATA_OS, PSENGINE_DELIVERY_TASK
+from polyswarm_engine.settings import (
+    PSENGINE_METADATA_ARCHTECTURE,
+    PSENGINE_METADATA_OS,
+    PSENGINE_DELIVERY_TASK,
+    PSENGINE_DISCARD_EXPIRED_BOUNTIES,
+)
 from polyswarm_engine.constants import (
     BENIGN,
     MALICIOUS,
@@ -245,11 +251,12 @@ class CeleryBackend:
         expiration = get_bounty_expiration(bounty)
         processing_start = datetime.now(timezone.utc)
         if processing_start > expiration:
-            raise exceptions.EngineTimeoutError(
-                'Current time %s is past expiration time %s',
-                processing_start.isoformat(),
-                expiration.isoformat(),
-            )
+            if PSENGINE_DISCARD_EXPIRED_BOUNTIES.upper() in ('1', 'YES', 'TRUE'):
+                raise exceptions.EngineTimeoutError(
+                    'Current time %s is past expiration time %s',
+                    processing_start.isoformat(),
+                    expiration.isoformat(),
+                )
 
         try:
             result = self._analyze(bounty)

polyswarm_engine/bidutils.py

@@ -44,7 +44,21 @@ def rescale_to_bid(bounty: 'Bounty', value: 't.SupportsInt', min=0, max=100) ->
 
 
 def dni_to_bid(bounty: 'Bounty', value: str) -> 'Bid':
-    """Transform string value from the None / Low / Med / High scale to a bid"""
+    """Transform string value from the None / Low / Med / High scale to a bid
+
+    From: https://www.dni.gov/files/documents/ICD/ICD_203_TA_Analytic_Standards_21_Dec_2022.pdf
+
+    | DNI Scale                               | Polyswarm Bid |
+    |-----------------------------------------+---------------|
+    | Not Specified                           | Not Specified |
+    | Almost No Chance / Remote               |             5 |
+    | Very Unlikely / Highly Improbable       |            15 |
+    | Unlikely / Improbable                   |            30 |
+    | Roughly Even Chance / Roughly Even Odds |            50 |
+    | Likely / Probable                       |            70 |
+    | Very Likely / Highly Probable           |            85 |
+    | Almost Certain / Nearly Certain         |            95 |to appropriate bid amounts.
+    """
     value = value.lower()
 
     if value == "none":

polyswarm_engine/celeryconfig.py

@@ -1,11 +1,20 @@
 import functools
 import importlib
+import io
+import os
+import sys
+import logging
 
+from celery.apps.worker import Worker
 from celery.worker.consumer import mingle, gossip
 from celery.worker.worker import WorkController
+import click
 
 import polyswarm_engine.settings
 
+logger = logging.getLogger(__name__)
+
+
 # monkey patch to enable -Ofair, the most stable during our tests
 _original_setup_defaults = WorkController.setup_defaults
 @functools.wraps(WorkController.setup_defaults)
@@ -17,6 +26,25 @@ WorkController.setup_defaults = _new_setup_defaults
 mingle.Mingle.compatible_transports = {}
 gossip.Gossip.compatible_transports = {}
 
+
+# monkey patch to make emit_banner use logging instead of print(sys.__stdout__)
+_original_emit_banner = Worker.emit_banner
+@functools.wraps(Worker.emit_banner)
+def _log_emit_banner(self):
+    original_stdout = sys.__stdout__
+    sys.__stdout__ = buffer = io.StringIO()
+    try:
+        _original_emit_banner(self)
+    finally:
+        sys.__stdout__ = original_stdout
+    banner_text = buffer.getvalue()
+    if os.getenv('LOG_FORMAT') == 'json':
+        # Celery outputs the banner as Cyan via ASCII terminal magic
+        banner_text = click.unstyle(banner_text)
+    logger.info(banner_text.lstrip())
+Worker.emit_banner = _log_emit_banner
+
+
 ##########################################
 # Celery Configuration
 ##########################################

polyswarm_engine/log_config.py

@@ -86,7 +86,7 @@ def get_logging(log_level=None, handler='console'):
             'click': {
                 'level': log_level,
                 'class': 'click_log.core.ClickHandler',
-                'formatter': 'click',
+                'formatter': 'click' if LOG_FORMAT == 'text' else LOG_FORMAT,
             },
         },
         'loggers': {

polyswarm_engine/settings.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 import os
 import platform
 import shutil
@@ -39,3 +40,6 @@ PSENGINE_TASK_ALWAYS_EAGER: bool = bool(int(os.getenv('PSENGINE_TASK_ALWAYS_EAGE
 #: Name of the Celery task that processes the delivery of assertions and votes.
 # If empty, fallback to doing the delivery directly via HTTP
 PSENGINE_DELIVERY_TASK = os.getenv('PSENGINE_DELIVERY_TASK', '')
+
+#: Turn on (set as 1) to discard bounties arrived after the expiration.
+PSENGINE_DISCARD_EXPIRED_BOUNTIES = os.getenv('PSENGINE_DISCARD_EXPIRED_BOUNTIES', '')

{polyswarm_engine-3.2.0.dist-info → polyswarm_engine-3.2.2.dist-info}/METADATA

@@ -1,10 +1,10 @@
 Metadata-Version: 2.4
 Name: polyswarm_engine
-Version: 3.2.0
+Version: 3.2.2
 Summary: Polyswarm engine libarary
 Author-email: Polyswarm Developers <developers@polyswarm.io>
 Project-URL: Homepage, https://github.com/polyswarm/polyswarm_engine
-Requires-Python: <4,>=3.10
+Requires-Python: <4,>=3.8
 Description-Content-Type: text/markdown
 Requires-Dist: celery
 Requires-Dist: click
@@ -12,6 +12,7 @@ Requires-Dist: click-log
 Requires-Dist: python-json-logger
 Requires-Dist: requests
 Provides-Extra: tests
+Requires-Dist: pathlib3x; python_version < "3.10" and extra == "tests"
 Requires-Dist: flake8; extra == "tests"
 Requires-Dist: isort; extra == "tests"
 Requires-Dist: mypy; extra == "tests"

{polyswarm_engine-3.2.0.dist-info → polyswarm_engine-3.2.2.dist-info}/RECORD

@@ -1,21 +1,21 @@
-polyswarm_engine/__init__.py,sha256=y9ToCb4bOwGo1f-RwO6eyxcxTke8P-QVBCULHEUEFes,838
-polyswarm_engine/backend.py,sha256=3Sf-nBPME8SAUHNE9Yuoa8Hw7cPr7o01iEjGmGXgxls,11387
-polyswarm_engine/bidutils.py,sha256=5Ipi5mAV2uxUp-TPUf39MmC81nIGBlyFyLIV-DWU03k,2111
+polyswarm_engine/__init__.py,sha256=B1P69VVyBIoRQSgfG9zjfp77zKYOsI81jw61pq-J5jw,838
+polyswarm_engine/backend.py,sha256=yqkduAdAMwxeKnfaQIaacwZMFyKlvwEgXKXqmA0r-X8,11580
+polyswarm_engine/bidutils.py,sha256=E-Oo7-LCCe5lNnoMKEmVDLW6lErqdkaTPfWoL-9YhJk,2881
 polyswarm_engine/bounty.py,sha256=gAHbApVqz07isKhzVSp4WX9PTic4Z0gTF_Lim9i0GE0,12627
-polyswarm_engine/celeryconfig.py,sha256=xirgl2YMru8CeTB3xpq3u0OoxBGoC7rEgW6_ytqePC0,2831
+polyswarm_engine/celeryconfig.py,sha256=7_mgzcegdgg5UOW_Iyach1HdsnpcVKr2uP8DZ9Npk-c,3610
 polyswarm_engine/cli.py,sha256=NwQmK1dFaROlfzrSqvQg5xPJVoDOC1-IWtH2xFrtirY,11278
 polyswarm_engine/command.py,sha256=RwdMpDeCvO0YM-7zKdej6o-8-zw_E8ltqxJ3MQvF2vw,915
 polyswarm_engine/constants.py,sha256=zeQRLSmYZH2kerzj5-S7ogsEloePreS2OsTQIw-lYeI,1379
 polyswarm_engine/engine.py,sha256=y7-wopHYTmCfyljJq4TbqKkOgz-Z5bQRvFv5liZhGcg,3789
 polyswarm_engine/exceptions.py,sha256=KEffKHrhX-2_ujoF9IBquMF5iq0amqP5uUkxoqkp31E,900
-polyswarm_engine/log_config.py,sha256=Dyzd30_zQtfRqPT7o2kQO1DgySzU4L1ykQDjUxnrbUk,3544
+polyswarm_engine/log_config.py,sha256=NnC4f5ukBW16LoXEskaIb98hi7LGZlOWv2F5OvuZf5A,3584
 polyswarm_engine/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-polyswarm_engine/settings.py,sha256=d200Oy0MKc2z6xFFhRuu9IU5BrrgbrjKCFY6QGBgAjw,1984
+polyswarm_engine/settings.py,sha256=kqANh7JTMWu8_FOf_5l2tLFANoSM9jDxjnNYMzg5HBA,2179
 polyswarm_engine/typing.py,sha256=Z6nroMW-xsR0uuJuUfsi7IQPEt_ydBa2pYUYTeFqUmI,3685
 polyswarm_engine/utils.py,sha256=2-ghShvJG8jsmNCpS4yv_1FH0r0pg2r67GvQru8v9aE,15492
 polyswarm_engine/wine.py,sha256=f01TEFv5Mvzv4_nGuEo1zYK19CJHAEFqJ4-8rva0xJA,4581
 polyswarm_engine/wsgi.py,sha256=9vv1WR0Rzcz7qYeReUrQw0ESHpK_3VbpMcIbmJiul-Q,4197
-polyswarm_engine-3.2.0.dist-info/METADATA,sha256=x7XZqf3KKq0fLKFlO2EVl75oxCXsKK_EPPcfstLQNYs,34233
-polyswarm_engine-3.2.0.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
-polyswarm_engine-3.2.0.dist-info/top_level.txt,sha256=iEEYOgxT_azXd38CWlfY_W0xOhxFBzBnwmfe62g96Pk,17
-polyswarm_engine-3.2.0.dist-info/RECORD,,
+polyswarm_engine-3.2.2.dist-info/METADATA,sha256=doXtO-XtWW7j4YV5X2Ro0MzwuiQns0sGQg2_3VXqklk,34303
+polyswarm_engine-3.2.2.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
+polyswarm_engine-3.2.2.dist-info/top_level.txt,sha256=iEEYOgxT_azXd38CWlfY_W0xOhxFBzBnwmfe62g96Pk,17
+polyswarm_engine-3.2.2.dist-info/RECORD,,