karton-core 5.7.0__py3-none-any.whl → 5.8.0__py3-none-any.whl

karton/core/karton.py

@@ -137,7 +137,7 @@ class Consumer(KartonServiceBase):
         )
         if self.task_timeout is None:
             self.task_timeout = self.config.getint("karton", "task_timeout")
-        self.current_task: Optional[Task] = None
+
         self._pre_hooks: List[Tuple[Optional[str], Callable[[Task], None]]] = []
         self._post_hooks: List[
             Tuple[
@@ -170,19 +170,18 @@ class Consumer(KartonServiceBase):
         """
 
         self.current_task = task
-        self.log_handler.set_task(self.current_task)
 
-        if not self.current_task.matches_filters(self.filters):
+        if not task.matches_filters(self.filters):
             self.log.info("Task rejected because binds are no longer valid.")
-            self.backend.set_task_status(self.current_task, TaskState.FINISHED)
+            self.backend.set_task_status(task, TaskState.FINISHED)
             # Task rejected: end of processing
             return
 
         exception_str = None
 
         try:
-            self.log.info("Received new task - %s", self.current_task.uid)
-            self.backend.set_task_status(self.current_task, TaskState.STARTED)
+            self.log.info("Received new task - %s", task.uid)
+            self.backend.set_task_status(task, TaskState.STARTED)
 
             self._run_pre_hooks()
 
@@ -190,22 +189,22 @@ class Consumer(KartonServiceBase):
             try:
                 if self.task_timeout:
                     with timeout(self.task_timeout):
-                        self.process(self.current_task)
+                        self.process(task)
                 else:
-                    self.process(self.current_task)
+                    self.process(task)
             except (Exception, TaskTimeoutError) as exc:
                 saved_exception = exc
                 raise
             finally:
                 self._run_post_hooks(saved_exception)
 
-            self.log.info("Task done - %s", self.current_task.uid)
+            self.log.info("Task done - %s", task.uid)
         except (Exception, TaskTimeoutError):
             exc_info = sys.exc_info()
             exception_str = traceback.format_exception(*exc_info)
 
             self.backend.increment_metrics(KartonMetrics.TASK_CRASHED, self.identity)
-            self.log.exception("Failed to process task - %s", self.current_task.uid)
+            self.log.exception("Failed to process task - %s", task.uid)
         finally:
             self.backend.increment_metrics(KartonMetrics.TASK_CONSUMED, self.identity)
 
@@ -215,9 +214,10 @@ class Consumer(KartonServiceBase):
             # if an exception was caught while processing
             if exception_str is not None:
                 task_state = TaskState.CRASHED
-                self.current_task.error = exception_str
+                task.error = exception_str
 
-            self.backend.set_task_status(self.current_task, task_state)
+            self.backend.set_task_status(task, task_state)
+            self.current_task = None
 
     @property
     def _bind(self) -> KartonBind:
@@ -228,6 +228,7 @@ class Consumer(KartonServiceBase):
             filters=self.filters,
             persistent=self.persistent,
             service_version=self.__class__.version,
+            is_async=False,
         )
 
     @classmethod

karton/core/logger.py

@@ -2,30 +2,32 @@ import logging
 import platform
 import traceback
 import warnings
-from typing import Optional
+from typing import Any, Callable, Dict
 
 from .backend import KartonBackend
-from .task import Task
+from .task import get_current_task
 
 HOSTNAME = platform.node()
 
 
-class KartonLogHandler(logging.Handler):
+class TaskContextFilter(logging.Filter):
     """
-    logging.Handler that passes logs to the Karton backend.
+    This is a filter which injects information about current task ID to the log.
     """
 
-    def __init__(self, backend: KartonBackend, channel: str) -> None:
-        logging.Handler.__init__(self)
-        self.backend = backend
-        self.task: Optional[Task] = None
-        self.is_consumer_active: bool = True
-        self.channel: str = channel
+    def filter(self, record: logging.LogRecord) -> bool:
+        current_task = get_current_task()
+        if current_task is not None:
+            record.task_id = current_task.task_uid
+        else:
+            record.task_id = "(no task)"
+        return True
 
-    def set_task(self, task: Task) -> None:
-        self.task = task
 
-    def emit(self, record: logging.LogRecord) -> None:
+class LogLineFormatterMixin:
+    format: Callable[[logging.LogRecord], str]
+
+    def prepare_log_line(self, record: logging.LogRecord) -> Dict[str, Any]:
         ignore_fields = [
             "args",
             "asctime",
@@ -54,11 +56,27 @@ class KartonLogHandler(logging.Handler):
         log_line["type"] = "log"
         log_line["message"] = self.format(record)
 
-        if self.task is not None:
-            log_line["task"] = self.task.serialize()
+        current_task = get_current_task()
+        if current_task is not None:
+            log_line["task"] = current_task.serialize()
 
         log_line["hostname"] = HOSTNAME
+        return log_line
+
+
+class KartonLogHandler(logging.Handler, LogLineFormatterMixin):
+    """
+    logging.Handler that passes logs to the Karton backend.
+    """
+
+    def __init__(self, backend: KartonBackend, channel: str) -> None:
+        logging.Handler.__init__(self)
+        self.backend = backend
+        self.is_consumer_active: bool = True
+        self.channel: str = channel
 
+    def emit(self, record: logging.LogRecord) -> None:
+        log_line = self.prepare_log_line(record)
         log_consumed = self.backend.produce_log(
             log_line, logger_name=self.channel, level=record.levelname
         )

karton/core/resource.py

@@ -150,34 +150,7 @@ class ResourceBase(object):
         }
 
 
-class LocalResource(ResourceBase):
-    """
-    Represents local resource with arbitrary binary data e.g. file contents.
-
-    Local resources will be uploaded to object hub (S3) during
-    task dispatching.
-
-    .. code-block:: python
-
-        # Creating resource from bytes
-        sample = Resource("original_name.exe", content=b"X5O!P%@AP[4\\
-        PZX54(P^)7CC)7}$EICAR-STANDARD-ANT...")
-
-        # Creating resource from path
-        sample = Resource("original_name.exe", path="sample/original_name.exe")
-
-    :param name: Name of the resource (e.g. name of file)
-    :param content: Resource content
-    :param path: Path of file with resource content
-    :param bucket: Alternative S3 bucket for resource
-    :param metadata: Resource metadata
-    :param uid: Alternative S3 resource id
-    :param sha256: Resource sha256 hash
-    :param fd: Seekable file descriptor
-    :param _flags: Resource flags
-    :param _close_fd: Close file descriptor after upload (default: False)
-    """
-
+class LocalResourceBase(ResourceBase):
     def __init__(
         self,
         name: str,
@@ -194,7 +167,7 @@ class LocalResource(ResourceBase):
         if len(list(filter(None, [path, content, fd]))) != 1:
             raise ValueError("You must exclusively provide a path, content or fd")
 
-        super(LocalResource, self).__init__(
+        super().__init__(
             name,
             content=content,
             path=path,
@@ -247,7 +220,7 @@ class LocalResource(ResourceBase):
         bucket: Optional[str] = None,
         metadata: Optional[Dict[str, Any]] = None,
         uid: Optional[str] = None,
-    ) -> "LocalResource":
+    ) -> "LocalResourceBase":
         """
         Resource extension, allowing to pass whole directory as a zipped resource.
 
@@ -305,6 +278,35 @@ class LocalResource(ResourceBase):
                 _close_fd=True,
             )
 
+
+class LocalResource(LocalResourceBase):
+    """
+    Represents local resource with arbitrary binary data e.g. file contents.
+
+    Local resources will be uploaded to object hub (S3) during
+    task dispatching.
+
+    .. code-block:: python
+
+        # Creating resource from bytes
+        sample = Resource("original_name.exe", content=b"X5O!P%@AP[4\\
+        PZX54(P^)7CC)7}$EICAR-STANDARD-ANT...")
+
+        # Creating resource from path
+        sample = Resource("original_name.exe", path="sample/original_name.exe")
+
+    :param name: Name of the resource (e.g. name of file)
+    :param content: Resource content
+    :param path: Path of file with resource content
+    :param bucket: Alternative S3 bucket for resource
+    :param metadata: Resource metadata
+    :param uid: Alternative S3 resource id
+    :param sha256: Resource sha256 hash
+    :param fd: Seekable file descriptor
+    :param _flags: Resource flags
+    :param _close_fd: Close file descriptor after upload (default: False)
+    """
+
     def _upload(self, backend: "KartonBackend") -> None:
         """Internal function for uploading resources
 

karton/core/task.py

@@ -3,6 +3,7 @@ import json
 import time
 import uuid
 import warnings
+from contextvars import ContextVar
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -24,6 +25,16 @@ if TYPE_CHECKING:
 
 import orjson
 
+current_task: ContextVar[Optional["Task"]] = ContextVar("current_task")
+
+
+def get_current_task() -> Optional["Task"]:
+    return current_task.get(None)
+
+
+def set_current_task(task: Optional["Task"]):
+    current_task.set(task)
+
 
 class TaskState(enum.Enum):
     DECLARED = "Declared"  # Task declared in TASKS_QUEUE
@@ -375,12 +386,15 @@ class Task(object):
         data: Union[str, bytes],
         backend: Optional["KartonBackend"] = None,
         parse_resources: bool = True,
+        resource_unserializer: Optional[Callable[[Dict], Any]] = None,
     ) -> "Task":
         """
         Unserialize Task instance from JSON string
 
         :param data: JSON-serialized task
-        :param backend: Backend instance to be bound to RemoteResource objects
+        :param backend: |
+            Backend instance to be bound to RemoteResource objects.
+            Deprecated: pass resource_unserializer instead.
         :param parse_resources: |
             If set to False (default is True), method doesn't
             deserialize '__karton_resource__' entries, which speeds up deserialization
@@ -388,6 +402,9 @@ class Task(object):
             filtering based on status.
             When resource deserialization is turned off, Task.unserialize will try
             to use faster 3rd-party JSON parser (orjson).
+        :param resource_unserializer: |
+            Resource factory used for deserialization of __karton_resource__
+            dictionary values.
         :return: Unserialized Task object
 
         :meta private:
@@ -399,7 +416,12 @@ class Task(object):
             RemoteResource object instances
             """
             if isinstance(value, dict) and "__karton_resource__" in value:
-                return RemoteResource.from_dict(value["__karton_resource__"], backend)
+                if resource_unserializer is None:
+                    return RemoteResource.from_dict(
+                        value["__karton_resource__"], backend
+                    )
+                else:
+                    return resource_unserializer(value["__karton_resource__"])
             return value
 
         if not isinstance(data, str):

{karton_core-5.7.0.dist-info → karton_core-5.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: karton-core
-Version: 5.7.0
+Version: 5.8.0
 Summary: Distributed malware analysis orchestration framework
 Home-page: https://github.com/CERT-Polska/karton
 Classifier: Programming Language :: Python :: 3
@@ -9,7 +9,8 @@ Classifier: License :: OSI Approved :: BSD License
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: boto3 <1.36.0
+Requires-Dist: aioboto3 ==14.3.0
+Requires-Dist: boto3 <1.37.4,>=1.37.2
 Requires-Dist: orjson
 Requires-Dist: redis
 

karton_core-5.8.0.dist-info/RECORD

@@ -0,0 +1,33 @@
+karton_core-5.8.0-nspkg.pth,sha256=vHa-jm6pBTeInFrmnsHMg9AOeD88czzQy-6QCFbpRcM,539
+karton/core/__init__.py,sha256=QuT0BWZyp799eY90tK3H1OD2hwuusqMJq8vQwpB3kG4,337
+karton/core/__version__.py,sha256=duDTiv1rL8Ee9_jtzzhpq-j4xkkpVPkUh2Daa_Ou-xA,22
+karton/core/backend.py,sha256=3tmjAM35jaoZr_5QqarcVH81LvKMEB63vyvMKCYXPCM,39934
+karton/core/base.py,sha256=6PEQiEWHjMtJKRj0dfgEEpNhgSRoKOBYF1WBlrmyBp0,9064
+karton/core/config.py,sha256=UTd0hLqYNUttfI7FYUDOJPaz-C3uj1Kw7xxataTG_OM,8234
+karton/core/exceptions.py,sha256=8i9WVzi4PinNlX10Cb-lQQC35Hl-JB5R_UKXa9AUKoQ,153
+karton/core/inspect.py,sha256=aIJQEOEkD5q2xLlV8nhxY5qL5zqcnprP-2DdP6ecKlE,6150
+karton/core/karton.py,sha256=4CISOmUTfaEaCJUtbYxJSBMzydT27o3a-R14VBNpmr0,15269
+karton/core/logger.py,sha256=UhYCoVARXaatvoJ2lO2mfBHeODOS7z8O-vqdeQhNmV4,2654
+karton/core/main.py,sha256=ir1-dhn3vbwfh2YHiM6ZYfRBbjwLvJSz0d8tuK1mb_4,8310
+karton/core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+karton/core/query.py,sha256=sf24DweVlXfJuBbBD_ns2LXhOV-IBwuPG3jBfTJu77s,12063
+karton/core/resource.py,sha256=GkU3JGSP1kOsAoblon4BWbQp31eZiDsCjaaDKanEokk,20872
+karton/core/task.py,sha256=QvAsSUIsyYiRmkgxgugrYkHWH2gczrFDfw2V4izdt1E,19566
+karton/core/test.py,sha256=cj6W4gNt0BpRjsYiiBt0hPE8dmRfUeIc8sSVkxB50cU,9123
+karton/core/utils.py,sha256=sEVqGdVPyYswWuVn8wYXBQmln8Az826N_2HgC__pmW8,4090
+karton/core/asyncio/__init__.py,sha256=ZgndeKzS3Yg2o8hebwFYJWlCRdW3ImdCOShK4EVmZ14,457
+karton/core/asyncio/backend.py,sha256=GF0z9YxWvUKYkvnBatCDthX0M9Kwt9cRfVSWQq5bc9E,12751
+karton/core/asyncio/base.py,sha256=YDNGyWzgVvt2TnfKvHYbJbcNJaQl95bdBq45YGEo-3Q,4246
+karton/core/asyncio/karton.py,sha256=LhzMGuJsmXdTEa323gZted8KgVfHH6l0j0_tTqMh4Z4,12932
+karton/core/asyncio/logger.py,sha256=BjkbuAeWylTmFjWv8-ckmOGf4nL2Tma96W0nIOc2vwk,1752
+karton/core/asyncio/resource.py,sha256=86AYm7JeVjEYRNw--h02HIS9xFvgddhktmDUp0qvTO4,12517
+karton/system/__init__.py,sha256=JF51OqRU_Y4c0unOulvmv1KzSHSq4ZpXU8ZsH4nefRM,63
+karton/system/__main__.py,sha256=QJkwIlSwaPRdzwKlNmCAL41HtDAa73db9MZKWmOfxGM,56
+karton/system/system.py,sha256=d_5hhLTthJdr_4gZEGQ6Y-kHvxeBqyQxjjx_wRs3xMA,17285
+karton_core-5.8.0.dist-info/LICENSE,sha256=o8h7hYhn7BJC_-DmrfqWwLjaR_Gbe0TZOOQJuN2ca3I,1519
+karton_core-5.8.0.dist-info/METADATA,sha256=26n1VrX-0ESRMG7fug5hqKxYT1URIa_NgypF5N1E2gg,6860
+karton_core-5.8.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+karton_core-5.8.0.dist-info/entry_points.txt,sha256=OgLlsXy61GP6-Yob3oXqeJ2hlRU6LBLj33fr0NufKz0,98
+karton_core-5.8.0.dist-info/namespace_packages.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
+karton_core-5.8.0.dist-info/top_level.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
+karton_core-5.8.0.dist-info/RECORD,,

karton_core-5.7.0.dist-info/RECORD

@@ -1,27 +0,0 @@
-karton_core-5.7.0-nspkg.pth,sha256=vHa-jm6pBTeInFrmnsHMg9AOeD88czzQy-6QCFbpRcM,539
-karton/core/__init__.py,sha256=QuT0BWZyp799eY90tK3H1OD2hwuusqMJq8vQwpB3kG4,337
-karton/core/__version__.py,sha256=QmHMXVnw5DVPfWzvN7FS1tOhDAesdxpM_aVOh9CMuSk,22
-karton/core/backend.py,sha256=_IOjN9pWdSBsDnTMYvg-Fpm6Ag-uf2Jb9LWmrtZqVAU,38773
-karton/core/base.py,sha256=lqVJvCHRMzvIOpS8SaWlOaSSJBEVkNQe0oClZC_GQYM,8225
-karton/core/config.py,sha256=M3dB0XgnUO5VzUcGyQa7FyKzmdgmDml1MrzG6CxEuvE,8100
-karton/core/exceptions.py,sha256=8i9WVzi4PinNlX10Cb-lQQC35Hl-JB5R_UKXa9AUKoQ,153
-karton/core/inspect.py,sha256=aIJQEOEkD5q2xLlV8nhxY5qL5zqcnprP-2DdP6ecKlE,6150
-karton/core/karton.py,sha256=l3joJWw8m23wlOErkcQmNFYhLFA5x2la6L0WopxJ7mk,15435
-karton/core/logger.py,sha256=J3XAyG88U0cwYC9zR6E3QD1uJenrQh7zS9-HgxhqeAs,2040
-karton/core/main.py,sha256=ir1-dhn3vbwfh2YHiM6ZYfRBbjwLvJSz0d8tuK1mb_4,8310
-karton/core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-karton/core/query.py,sha256=sf24DweVlXfJuBbBD_ns2LXhOV-IBwuPG3jBfTJu77s,12063
-karton/core/resource.py,sha256=9kWXpMBRfudH0_whJfSSI27K3Gwv2u93CVa7p68Q5UM,20842
-karton/core/task.py,sha256=gW1szMi5PN2Y06X-Ryo7cmEVluZv1r7W5tvmwIJiD94,18808
-karton/core/test.py,sha256=cj6W4gNt0BpRjsYiiBt0hPE8dmRfUeIc8sSVkxB50cU,9123
-karton/core/utils.py,sha256=sEVqGdVPyYswWuVn8wYXBQmln8Az826N_2HgC__pmW8,4090
-karton/system/__init__.py,sha256=JF51OqRU_Y4c0unOulvmv1KzSHSq4ZpXU8ZsH4nefRM,63
-karton/system/__main__.py,sha256=QJkwIlSwaPRdzwKlNmCAL41HtDAa73db9MZKWmOfxGM,56
-karton/system/system.py,sha256=d_5hhLTthJdr_4gZEGQ6Y-kHvxeBqyQxjjx_wRs3xMA,17285
-karton_core-5.7.0.dist-info/LICENSE,sha256=o8h7hYhn7BJC_-DmrfqWwLjaR_Gbe0TZOOQJuN2ca3I,1519
-karton_core-5.7.0.dist-info/METADATA,sha256=MrmtycTaYsNB8v0LRyuLIHL2bV17n1Lt6e-ak4RfrH8,6818
-karton_core-5.7.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-karton_core-5.7.0.dist-info/entry_points.txt,sha256=OgLlsXy61GP6-Yob3oXqeJ2hlRU6LBLj33fr0NufKz0,98
-karton_core-5.7.0.dist-info/namespace_packages.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
-karton_core-5.7.0.dist-info/top_level.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
-karton_core-5.7.0.dist-info/RECORD,,