karton-core 5.3.3__py3-none-any.whl → 5.4.0__py3-none-any.whl

karton/core/__version__.py

@@ -1 +1 @@
-__version__ = "5.3.3"
+__version__ = "5.4.0"

karton/core/backend.py

@@ -449,9 +449,7 @@ class KartonBackend:
             chunk_size,
         )
         return [
-            Task.unserialize(task_data, backend=self)
-            if parse_resources
-            else Task.unserialize(task_data, parse_resources=False)
+            Task.unserialize(task_data, backend=self, parse_resources=parse_resources)
             for chunk in keys
             for task_data in self.redis.mget(chunk)
             if task_data is not None
@@ -465,9 +463,9 @@ class KartonBackend:
     ) -> Iterator[Task]:
         for chunk in chunks_iter(task_keys, chunk_size):
             yield from (
-                Task.unserialize(task_data, backend=self)
-                if parse_resources
-                else Task.unserialize(task_data, parse_resources=False)
+                Task.unserialize(
+                    task_data, backend=self, parse_resources=parse_resources
+                )
                 for task_data in self.redis.mget(chunk)
                 if task_data is not None
             )
@@ -534,6 +532,58 @@ class KartonBackend:
             self.iter_all_tasks(chunk_size=chunk_size, parse_resources=parse_resources)
         )
 
+    def _iter_legacy_task_tree(
+        self, root_uid: str, chunk_size: int = 1000, parse_resources: bool = True
+    ) -> Iterator[Task]:
+        """
+        Processes tasks made by <5.4.0 (unrouted from <5.4.0 producers or existing
+        before upgrade)
+
+        Used internally by iter_task_tree.
+        """
+        # Iterate over all karton tasks that do not match the new task id format
+        legacy_task_keys = self.redis.scan_iter(
+            match=f"{KARTON_TASK_NAMESPACE}:[^{{]*", count=chunk_size
+        )
+        for chunk in chunks_iter(legacy_task_keys, chunk_size):
+            yield from filter(
+                lambda task: task.root_uid == root_uid,
+                (
+                    Task.unserialize(
+                        task_data, backend=self, parse_resources=parse_resources
+                    )
+                    for task_data in self.redis.mget(chunk)
+                    if task_data is not None
+                ),
+            )
+
+    def iter_task_tree(
+        self, root_uid: str, chunk_size: int = 1000, parse_resources: bool = True
+    ) -> Iterator[Task]:
+        """
+        Iterates all tasks that belong to the same analysis task tree
+        and have the same root_uid
+
+        :param root_uid: Root identifier of task tree
+        :param chunk_size: Size of chunks passed to the Redis SCAN and MGET command
+        :param parse_resources: If set to False, resources are not parsed.
+            It speeds up deserialization. Read :py:meth:`Task.unserialize` documentation
+            to learn more.
+        :return: Iterator with task objects
+        """
+        # Process <5.4.0 tasks (unrouted from <5.4.0 producers
+        # or existing before upgrade)
+        yield from self._iter_legacy_task_tree(
+            root_uid, chunk_size=chunk_size, parse_resources=parse_resources
+        )
+        # Process >=5.4.0 tasks
+        task_keys = self.redis.scan_iter(
+            match=f"{KARTON_TASK_NAMESPACE}:{{{root_uid}}}:*", count=chunk_size
+        )
+        yield from self._iter_tasks(
+            task_keys, chunk_size=chunk_size, parse_resources=parse_resources
+        )
+
     def register_task(self, task: Task, pipe: Optional[Pipeline] = None) -> None:
         """
         Register or update task in Redis.

karton/core/inspect.py

@@ -1,5 +1,5 @@
 from collections import defaultdict
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 from .backend import KartonBackend, KartonBind
 from .task import Task, TaskState
@@ -9,9 +9,9 @@ class KartonQueue:
     """
     View object representing a Karton queue
 
-    :param bind: :py:meth:`KartonBind` object representing the queue bind
+    :param bind: :class:`KartonBind` object representing the queue bind
     :param tasks: List of tasks currently in queue
-    :param state: :py:meth:`KartonBackend` object to be used
+    :param state: :class:`KartonState` object to be used
     """
 
     def __init__(
@@ -48,7 +48,7 @@ class KartonAnalysis:
 
     :param root_uid: Analysis root task uid
     :param tasks: List of tasks
-    :param state: :py:meth:`KartonBackend` object to be used
+    :param state: :class:`KartonState` object to be used
     """
 
     def __init__(self, root_uid: str, tasks: List[Task], state: "KartonState") -> None:
@@ -89,7 +89,7 @@ def get_queues_for_tasks(
     Group task objects by their queue name
 
     :param tasks: Task objects to group
-    :param state: :py:meth:`KartonBackend` to bind to created queues
+    :param state: :class:`KartonState` object to be used
     :return: A dictionary containing the queue names and lists of tasks
     """
     tasks_per_queue = defaultdict(list)
@@ -119,30 +119,68 @@ class KartonState:
     :param backend: :py:meth:`KartonBackend` object to use for data fetching
     """
 
-    def __init__(self, backend: KartonBackend) -> None:
+    def __init__(self, backend: KartonBackend, parse_resources: bool = False) -> None:
         self.backend = backend
         self.binds = {bind.identity: bind for bind in backend.get_binds()}
         self.replicas = backend.get_online_consumers()
-        self.tasks = backend.get_all_tasks()
-        self.pending_tasks = [
-            task for task in self.tasks if task.status != TaskState.FINISHED
-        ]
-
-        # Tasks grouped by root_uid
-        tasks_per_analysis = defaultdict(list)
-
-        for task in self.pending_tasks:
-            tasks_per_analysis[task.root_uid].append(task)
-
-        self.analyses = {
-            root_uid: KartonAnalysis(root_uid=root_uid, tasks=tasks, state=self)
-            for root_uid, tasks in tasks_per_analysis.items()
-        }
-        queues = get_queues_for_tasks(self.pending_tasks, self)
-        # Present registered queues without tasks
-        for bind_name, bind in self.binds.items():
-            if bind_name not in queues:
-                queues[bind_name] = KartonQueue(
-                    bind=self.binds[bind_name], tasks=[], state=self
+        self.parse_resources = parse_resources
+
+        self._tasks: Optional[List[Task]] = None
+        self._pending_tasks: Optional[List[Task]] = None
+        self._analyses: Optional[Dict[str, KartonAnalysis]] = None
+        self._queues: Optional[Dict[str, KartonQueue]] = None
+
+    @property
+    def tasks(self) -> List[Task]:
+        if self._tasks is None:
+            self._tasks = self.backend.get_all_tasks(
+                parse_resources=self.parse_resources
+            )
+        return self._tasks
+
+    @property
+    def pending_tasks(self) -> List[Task]:
+        if self._pending_tasks is None:
+            self._pending_tasks = [
+                task for task in self.tasks if task.status != TaskState.FINISHED
+            ]
+        return self._pending_tasks
+
+    @property
+    def analyses(self) -> Dict[str, KartonAnalysis]:
+        if self._analyses is None:
+            # Tasks grouped by root_uid
+            tasks_per_analysis = defaultdict(list)
+
+            for task in self.pending_tasks:
+                tasks_per_analysis[task.root_uid].append(task)
+
+            self._analyses = {
+                root_uid: KartonAnalysis(root_uid=root_uid, tasks=tasks, state=self)
+                for root_uid, tasks in tasks_per_analysis.items()
+            }
+        return self._analyses
+
+    @property
+    def queues(self) -> Dict[str, KartonQueue]:
+        if self._queues is None:
+            queues = get_queues_for_tasks(self.pending_tasks, self)
+            # Present registered queues without tasks
+            for bind_name, bind in self.binds.items():
+                if bind_name not in queues:
+                    queues[bind_name] = KartonQueue(
+                        bind=self.binds[bind_name], tasks=[], state=self
+                    )
+            self._queues = queues
+        return self._queues
+
+    def get_analysis(self, root_uid: str) -> KartonAnalysis:
+        return KartonAnalysis(
+            root_uid=root_uid,
+            tasks=list(
+                self.backend.iter_task_tree(
+                    root_uid, parse_resources=self.parse_resources
                 )
-        self.queues = queues
+            ),
+            state=self,
+        )

karton/core/karton.py

@@ -12,6 +12,7 @@ from .__version__ import __version__
 from .backend import KartonBackend, KartonBind, KartonMetrics
 from .base import KartonBase, KartonServiceBase
 from .config import Config
+from .exceptions import TaskTimeoutError
 from .resource import LocalResource
 from .task import Task, TaskState
 from .utils import timeout
@@ -129,7 +130,10 @@ class Consumer(KartonServiceBase):
         self.current_task: Optional[Task] = None
         self._pre_hooks: List[Tuple[Optional[str], Callable[[Task], None]]] = []
         self._post_hooks: List[
-            Tuple[Optional[str], Callable[[Task, Optional[Exception]], None]]
+            Tuple[
+                Optional[str],
+                Callable[[Task, Optional[BaseException]], None],
+            ]
         ] = []
 
     @abc.abstractmethod
@@ -179,14 +183,14 @@ class Consumer(KartonServiceBase):
                         self.process(self.current_task)
                 else:
                     self.process(self.current_task)
-            except Exception as exc:
+            except (Exception, TaskTimeoutError) as exc:
                 saved_exception = exc
                 raise
             finally:
                 self._run_post_hooks(saved_exception)
 
             self.log.info("Task done - %s", self.current_task.uid)
-        except Exception:
+        except (Exception, TaskTimeoutError):
             exc_info = sys.exc_info()
             exception_str = traceback.format_exception(*exc_info)
 
@@ -260,7 +264,7 @@ class Consumer(KartonServiceBase):
 
     def add_post_hook(
         self,
-        callback: Callable[[Task, Optional[Exception]], None],
+        callback: Callable[[Task, Optional[BaseException]], None],
         name: Optional[str] = None,
     ) -> None:
         """
@@ -289,7 +293,7 @@ class Consumer(KartonServiceBase):
                 else:
                     self.log.exception("Pre-hook failed")
 
-    def _run_post_hooks(self, exception: Optional[Exception]) -> None:
+    def _run_post_hooks(self, exception: Optional[BaseException]) -> None:
         """
         Run registered postprocessing hooks
 
@@ -431,7 +435,7 @@ class Karton(Consumer, Producer):
         self._send_signaling_status_task("task_begin")
 
     def _send_signaling_status_task_end(
-        self, task: Task, ex: Optional[Exception]
+        self, task: Task, ex: Optional[BaseException]
     ) -> None:
         """Send a begin status signaling task.
 

karton/core/task.py

@@ -106,13 +106,18 @@ class Task(object):
             raise ValueError("Persistent headers should be an instance of a dict")
 
         if uid is None:
-            self.uid = str(uuid.uuid4())
+            task_uid = str(uuid.uuid4())
+            if root_uid is None:
+                self.root_uid = task_uid
+            else:
+                self.root_uid = root_uid
+            # New-style UID format introduced in v5.4.0
+            # {12345678-1234-1234-1234-12345678abcd}:12345678-1234-1234-1234-12345678abcd
+            self.uid = f"{{{self.root_uid}}}:{task_uid}"
         else:
             self.uid = uid
-
-        if root_uid is None:
-            self.root_uid = self.uid
-        else:
+            if root_uid is None:
+                raise ValueError("root_uid cannot be None when uid is not None")
             self.root_uid = root_uid
 
         self.orig_uid = orig_uid
@@ -137,6 +142,21 @@ class Task(object):
     def receiver(self) -> Optional[str]:
         return self.headers.get("receiver")
 
+    @property
+    def task_uid(self) -> str:
+        return self.fquid_to_uid(self.uid)
+
+    @staticmethod
+    def fquid_to_uid(fquid: str) -> str:
+        """
+        Gets task uid from fully-qualified fquid ({root_uid}:task_uid)
+
+        :return: Task uid
+        """
+        if ":" not in fquid:
+            return fquid
+        return fquid.split(":")[-1]
+
     def fork_task(self) -> "Task":
         """
         Fork task to transfer single task to many queues (but use different UID).
@@ -212,38 +232,65 @@ class Task(object):
         :meta private:
         """
 
-        matches = False
-        for task_filter in filters:
-            matched = []
-            for filter_key, filter_value in task_filter.items():
-                # Coerce to string for comparison
-                header_value = self.headers.get(filter_key)
+        def test_filter(headers: Dict[str, Any], filter: Dict[str, Any]) -> int:
+            """
+            Filter match follows AND logic, but it's non-boolean because filters may be
+            negated (task:!platform).
+
+            Result values are as follows:
+            - 1  - positive match, no mismatched values in headers
+                   (all matched)
+            - 0  - no match, found value that doesn't match to the filter
+                   (some are not matched)
+            - -1 - negative match, found value that matches negated filter value
+                   (all matched but found negative matches)
+            """
+            matches = 1
+            for filter_key, filter_value in filter.items():
+                # Coerce filter value to string
                 filter_value_str = str(filter_value)
-                header_value_str = str(header_value)
-
                 negated = False
                 if filter_value_str.startswith("!"):
                     negated = True
                     filter_value_str = filter_value_str[1:]
 
-                if header_value is None:
-                    matched.append(negated)
-                    continue
+                # If expected key doesn't exist in headers
+                if filter_key not in headers:
+                    # Negated filter ignores non-existent values
+                    if negated:
+                        continue
+                    # But positive filter doesn't
+                    return 0
 
+                # Coerce header value to string
+                header_value_str = str(headers[filter_key])
                 # fnmatch is great for handling simple wildcard patterns (?, *, [abc])
                 match = fnmatch.fnmatchcase(header_value_str, filter_value_str)
-                # if matches, but it's negated then we can return straight away
-                # since no matter the other filters
+                # If matches, but it's negated: it's negative match
                 if match and negated:
-                    return False
-
-                # else, apply a XOR logic to take care of negation matching
-                matched.append(match != negated)
-
-            # set the flag if all consumer filter fields match the task header.
-            # It will be set to True only if at least one filter matches the header
-            matches |= all(matched)
-
+                    matches = -1
+                # If doesn't match but filter is not negated: it's not a match
+                if not match and not negated:
+                    return 0
+            # If there are no mismatched values: filter is matched
+            return matches
+
+        # List of filter matches follow OR logic, but -1 is special
+        # If there is any -1, result is False
+        #   (any matched, but it's negative match)
+        # If there is any 1, but no -1's: result is True
+        #   (any matched, no negative match)
+        # If there are only 0's: result is False
+        #   (none matched)
+        matches = False
+        for task_filter in filters:
+            match_result = test_filter(self.headers, task_filter)
+            if match_result == -1:
+                # Any negative match results in False
+                return False
+            if match_result == 1:
+                # Any positive match but without negative matches results in True
+                matches = True
         return matches
 
     def set_task_parent(self, parent: "Task"):
@@ -407,9 +454,7 @@ class Task(object):
             process. This flag is used mainly for multiple task processing e.g.
             filtering based on status.
             When resource deserialization is turned off, Task.unserialize will try
-            to use faster 3rd-party JSON parser (orjson) if it's installed. It's not
-            added as a required dependency but can speed up things if you need to check
-            status of multiple tasks at once.
+            to use faster 3rd-party JSON parser (orjson).
         :return: Unserialized Task object
 
         :meta private:
@@ -430,11 +475,7 @@ class Task(object):
         if parse_resources:
             task_data = json.loads(data, object_hook=unserialize_resources)
         else:
-            try:
-                task_data = orjson.loads(data)
-            except orjson.JSONDecodeError:
-                # fallback, in case orjson raises exception during loading
-                task_data = json.loads(data, object_hook=unserialize_resources)
+            task_data = orjson.loads(data)
 
         # Compatibility with Karton <5.2.0
         headers_persistent_fallback = task_data["payload_persistent"].get(

karton/system/system.py

@@ -103,7 +103,7 @@ class SystemService(KartonServiceBase):
                 and current_time > task.last_update + self.task_dispatched_timeout
             ):
                 to_delete.append(task)
-                self.log.warning(
+                self.log.error(
                     "Task %s is in Dispatched state more than %d seconds. "
                     "Killed. (origin: %s)",
                     task.uid,
@@ -116,7 +116,7 @@ class SystemService(KartonServiceBase):
                 and current_time > task.last_update + self.task_started_timeout
             ):
                 to_delete.append(task)
-                self.log.warning(
+                self.log.error(
                     "Task %s is in Started state more than %d seconds. "
                     "Killed. (receiver: %s)",
                     task.uid,
@@ -166,7 +166,7 @@ class SystemService(KartonServiceBase):
 
     def route_task(self, task: Task, binds: List[KartonBind]) -> None:
         # Performs routing of task
-        self.log.info("[%s] Processing task %s", task.root_uid, task.uid)
+        self.log.info("[%s] Processing task %s", task.root_uid, task.task_uid)
         # store the producer-task relationship in redis for task tracking
         self.backend.log_identity_output(
             task.headers.get("origin", "unknown"), task.headers

{karton_core-5.3.3.dist-info → karton_core-5.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: karton-core
-Version: 5.3.3
+Version: 5.4.0
 Summary: Distributed malware analysis orchestration framework
 Home-page: https://github.com/CERT-Polska/karton
 License: UNKNOWN

{karton_core-5.3.3.dist-info → karton_core-5.4.0.dist-info}/RECORD

@@ -1,26 +1,26 @@
-karton_core-5.3.3-nspkg.pth,sha256=vHa-jm6pBTeInFrmnsHMg9AOeD88czzQy-6QCFbpRcM,539
+karton_core-5.4.0-nspkg.pth,sha256=vHa-jm6pBTeInFrmnsHMg9AOeD88czzQy-6QCFbpRcM,539
 karton/core/__init__.py,sha256=QuT0BWZyp799eY90tK3H1OD2hwuusqMJq8vQwpB3kG4,337
-karton/core/__version__.py,sha256=1nSgbQKO6KlKoDYLTNXC8iJUIO_SuORsK-CISssv5l0,22
-karton/core/backend.py,sha256=evOzlz1v1sxWusc8VojGAYyeyi9fcbVoEPm6WoNT1Xs,34696
+karton/core/__version__.py,sha256=xjYaBGUFGg0kGZj_WhuoFyPD8NILPsr79SaMwmYQGSg,22
+karton/core/backend.py,sha256=-sQG7utnaWLJOEcafeSwEDLnkflPqtSCwg_mn_nnFhg,36727
 karton/core/base.py,sha256=C6Lco3E0XCsxvEjeVOLR9fxh_IWJ1vjC9BqUYsQyewE,8083
 karton/core/config.py,sha256=7oKchitq6pWzPuXRfjBXqVT_BgGIz2p-CDo1RGaNJQg,8118
 karton/core/exceptions.py,sha256=8i9WVzi4PinNlX10Cb-lQQC35Hl-JB5R_UKXa9AUKoQ,153
-karton/core/inspect.py,sha256=rIa0u4u12vG_RudPfc9UAS4RZD56W8qbUa8n1dDIkX0,4868
-karton/core/karton.py,sha256=-BWsNvbL_yFFlSPjEOXlBqkENEWLTfIB0ETjowh7Mjo,14863
+karton/core/inspect.py,sha256=aIJQEOEkD5q2xLlV8nhxY5qL5zqcnprP-2DdP6ecKlE,6150
+karton/core/karton.py,sha256=9SOAviG42kSsPqc3EuaHzWtA_KywMtc01hmU6FaJpHo,15007
 karton/core/logger.py,sha256=J3XAyG88U0cwYC9zR6E3QD1uJenrQh7zS9-HgxhqeAs,2040
 karton/core/main.py,sha256=ir1-dhn3vbwfh2YHiM6ZYfRBbjwLvJSz0d8tuK1mb_4,8310
 karton/core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 karton/core/resource.py,sha256=tA3y_38H9HVKIrCeAU70zHUkQUv0BuCQWMC470JLxxc,20321
-karton/core/task.py,sha256=1yyCH6A6MBJ7yuKECmsLrZDRECbKQ-48dJ31ZL7FR5Y,19644
+karton/core/task.py,sha256=diwg8uUl57NEYNRjT1l5CPiNw3EQcU11BnrLul33fx0,21350
 karton/core/test.py,sha256=tms-YM7sUKQDHN0vm2_W7DIvHnO_ld_VPsWHnsbKSfk,9102
 karton/core/utils.py,sha256=sEVqGdVPyYswWuVn8wYXBQmln8Az826N_2HgC__pmW8,4090
 karton/system/__init__.py,sha256=JF51OqRU_Y4c0unOulvmv1KzSHSq4ZpXU8ZsH4nefRM,63
 karton/system/__main__.py,sha256=QJkwIlSwaPRdzwKlNmCAL41HtDAa73db9MZKWmOfxGM,56
-karton/system/system.py,sha256=AsQLGF_8YI-o4dCOe2hyQefYlhGEguFSoPg1EcQtJyU,13791
-karton_core-5.3.3.dist-info/LICENSE,sha256=o8h7hYhn7BJC_-DmrfqWwLjaR_Gbe0TZOOQJuN2ca3I,1519
-karton_core-5.3.3.dist-info/METADATA,sha256=sumt5njKhDPynjbmFZvPxdPPSCCXR4wr3xchvowJPC8,6847
-karton_core-5.3.3.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-karton_core-5.3.3.dist-info/entry_points.txt,sha256=FJj5EZuvFP0LkagjX_dLbRGBUnuLjgBiSyiFfq4c86U,99
-karton_core-5.3.3.dist-info/namespace_packages.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
-karton_core-5.3.3.dist-info/top_level.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
-karton_core-5.3.3.dist-info/RECORD,,
+karton/system/system.py,sha256=yF_d71a8w7JYA7IXUt63d5_QBH6x1QplB-xcrzQTXL4,13792
+karton_core-5.4.0.dist-info/LICENSE,sha256=o8h7hYhn7BJC_-DmrfqWwLjaR_Gbe0TZOOQJuN2ca3I,1519
+karton_core-5.4.0.dist-info/METADATA,sha256=kopeYFCI9EoFQbc7J7woZWjI_5egy29-lYUW7UzEQ2I,6847
+karton_core-5.4.0.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+karton_core-5.4.0.dist-info/entry_points.txt,sha256=FJj5EZuvFP0LkagjX_dLbRGBUnuLjgBiSyiFfq4c86U,99
+karton_core-5.4.0.dist-info/namespace_packages.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
+karton_core-5.4.0.dist-info/top_level.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
+karton_core-5.4.0.dist-info/RECORD,,