karton-core 5.4.0__py3-none-any.whl → 5.5.1__py3-none-any.whl

karton/core/__version__.py

@@ -1 +1 @@
-__version__ = "5.4.0"
+__version__ = "5.5.1"

karton/core/karton.py

@@ -8,6 +8,7 @@ import time
 import traceback
 from typing import Any, Callable, Dict, List, Optional, Tuple, cast
 
+from . import query
 from .__version__ import __version__
 from .backend import KartonBackend, KartonBind, KartonMetrics
 from .base import KartonBase, KartonServiceBase
@@ -122,6 +123,9 @@ class Consumer(KartonServiceBase):
         if self.filters is None:
             raise ValueError("Cannot bind consumer on Empty binds")
 
+        # Dummy conversion to make sure the filters are well-formed.
+        query.convert(self.filters)
+
         self.persistent = (
             self.config.getboolean("karton", "persistent", self.persistent)
             and not self.debug

karton/core/query.py

@@ -0,0 +1,350 @@
+import fnmatch
+import re
+from collections.abc import Mapping, Sequence
+from typing import Dict, Type
+
+# Source code adopted from https://github.com/kapouille/mongoquery
+# Original licenced under "The Unlicense" license.
+
+
+class QueryError(Exception):
+    """Query error exception"""
+
+    pass
+
+
+class _Undefined(object):
+    pass
+
+
+def is_non_string_sequence(entry):
+    """Returns True if entry is a Python sequence iterable, and not a string"""
+    return isinstance(entry, Sequence) and not isinstance(entry, str)
+
+
+class Query(object):
+    """The Query class is used to match an object against a MongoDB-like query"""
+
+    def __init__(self, definition):
+        self._definition = definition
+
+    def match(self, entry):
+        """Matches the entry object against the query specified on instanciation"""
+        return self._match(self._definition, entry)
+
+    def _match(self, condition, entry):
+        if isinstance(condition, Mapping):
+            return all(
+                self._process_condition(sub_operator, sub_condition, entry)
+                for sub_operator, sub_condition in condition.items()
+            )
+        if is_non_string_sequence(entry):
+            return condition in entry
+        return condition == entry
+
+    def _extract(self, entry, path):
+        if not path:
+            return entry
+        if entry is None:
+            return entry
+        if is_non_string_sequence(entry):
+            try:
+                index = int(path[0])
+                return self._extract(entry[index], path[1:])
+            except ValueError:
+                return [self._extract(item, path) for item in entry]
+        elif isinstance(entry, Mapping) and path[0] in entry:
+            return self._extract(entry[path[0]], path[1:])
+        else:
+            return _Undefined()
+
+    def _path_exists(self, operator, condition, entry):
+        keys_list = list(operator.split("."))
+        for i, k in enumerate(keys_list):
+            if isinstance(entry, Sequence) and not k.isdigit():
+                for elem in entry:
+                    operator = ".".join(keys_list[i:])
+                    if self._path_exists(operator, condition, elem) == condition:
+                        return condition
+                return not condition
+            elif isinstance(entry, Sequence):
+                k = int(k)
+            try:
+                entry = entry[k]
+            except (TypeError, IndexError, KeyError):
+                return not condition
+        return condition
+
+    def _process_condition(self, operator, condition, entry):
+        if isinstance(condition, Mapping) and "$exists" in condition:
+            if isinstance(operator, str) and operator.find(".") != -1:
+                return self._path_exists(operator, condition["$exists"], entry)
+            elif condition["$exists"] != (operator in entry):
+                return False
+            elif tuple(condition.keys()) == ("$exists",):
+                return True
+        if isinstance(operator, str):
+            if operator.startswith("$"):
+                try:
+                    return getattr(self, "_" + operator[1:])(condition, entry)
+                except AttributeError:
+                    raise QueryError(f"{operator} operator isn't supported")
+            else:
+                try:
+                    extracted_data = self._extract(entry, operator.split("."))
+                except IndexError:
+                    extracted_data = _Undefined()
+        else:
+            if operator not in entry:
+                return False
+            extracted_data = entry[operator]
+        return self._match(condition, extracted_data)
+
+    @staticmethod
+    def _not_implemented(*_):
+        raise NotImplementedError
+
+    @staticmethod
+    def _noop(*_):
+        return True
+
+    @staticmethod
+    def _eq(condition, entry):
+        try:
+            return entry == condition
+        except TypeError:
+            return False
+
+    @staticmethod
+    def _gt(condition, entry):
+        try:
+            return entry > condition
+        except TypeError:
+            return False
+
+    @staticmethod
+    def _gte(condition, entry):
+        try:
+            return entry >= condition
+        except TypeError:
+            return False
+
+    @staticmethod
+    def _in(condition, entry):
+        if is_non_string_sequence(condition):
+            for elem in condition:
+                if is_non_string_sequence(entry) and elem in entry:
+                    return True
+                elif not is_non_string_sequence(entry) and elem == entry:
+                    return True
+            return False
+        else:
+            raise TypeError("condition must be a list")
+
+    @staticmethod
+    def _lt(condition, entry):
+        try:
+            return entry < condition
+        except TypeError:
+            return False
+
+    @staticmethod
+    def _lte(condition, entry):
+        try:
+            return entry <= condition
+        except TypeError:
+            return False
+
+    @staticmethod
+    def _ne(condition, entry):
+        return entry != condition
+
+    def _nin(self, condition, entry):
+        return not self._in(condition, entry)
+
+    def _and(self, condition, entry):
+        if isinstance(condition, Sequence):
+            return all(self._match(sub_condition, entry) for sub_condition in condition)
+        raise QueryError(f"$and has been attributed incorrect argument {condition}")
+
+    def _nor(self, condition, entry):
+        if isinstance(condition, Sequence):
+            return all(
+                not self._match(sub_condition, entry) for sub_condition in condition
+            )
+        raise QueryError(f"$nor has been attributed incorrect argument {condition}")
+
+    def _not(self, condition, entry):
+        return not self._match(condition, entry)
+
+    def _or(self, condition, entry):
+        if isinstance(condition, Sequence):
+            return any(self._match(sub_condition, entry) for sub_condition in condition)
+        raise QueryError(f"$or has been attributed incorrect argument {condition}")
+
+    @staticmethod
+    def _type(condition, entry):
+        bson_type: Dict[int, Type] = {
+            1: float,
+            2: str,
+            3: Mapping,
+            4: Sequence,
+            5: bytearray,
+            7: str,  # object id (uuid)
+            8: bool,
+            9: str,  # date (UTC datetime)
+            10: type(None),
+            11: re.Pattern,  # regex,
+            13: str,  # Javascript
+            15: str,  # JavaScript (with scope)
+            16: int,  # 32-bit integer
+            17: int,  # Timestamp
+            18: int,  # 64-bit integer
+        }
+        bson_alias = {
+            "double": 1,
+            "string": 2,
+            "object": 3,
+            "array": 4,
+            "binData": 5,
+            "objectId": 7,
+            "bool": 8,
+            "date": 9,
+            "null": 10,
+            "regex": 11,
+            "javascript": 13,
+            "javascriptWithScope": 15,
+            "int": 16,
+            "timestamp": 17,
+            "long": 18,
+        }
+
+        if condition == "number":
+            return any(
+                [
+                    isinstance(entry, bson_type[bson_alias[alias]])
+                    for alias in ["double", "int", "long"]
+                ]
+            )
+
+        # resolves bson alias, or keeps original condition value
+        condition = bson_alias.get(condition, condition)
+
+        if condition not in bson_type:
+            raise QueryError(f"$type has been used with unknown type {condition}")
+
+        return isinstance(entry, bson_type[condition])
+
+    _exists = _noop
+
+    @staticmethod
+    def _mod(condition, entry):
+        return entry % condition[0] == condition[1]
+
+    @staticmethod
+    def _regex(condition, entry):
+        if not isinstance(entry, str):
+            return False
+        # If the caller has supplied a compiled regex, assume options are already
+        # included.
+        if isinstance(condition, re.Pattern):
+            return bool(re.search(condition, entry))
+
+        try:
+            regex = re.match(r"\A/(.+)/([imsx]{,4})\Z", condition, flags=re.DOTALL)
+        except TypeError:
+            raise QueryError(
+                f"{condition} is not a regular expression and should be a string"
+            )
+
+        flags = 0
+        if regex:
+            options = regex.group(2)
+            for option in options:
+                flags |= getattr(re, option.upper())
+            exp = regex.group(1)
+        else:
+            exp = condition
+
+        try:
+            match = re.search(exp, entry, flags=flags)
+        except Exception as error:
+            raise QueryError(f"{condition} failed to execute with error {error!r}")
+        return bool(match)
+
+    _options = _text = _where = _not_implemented
+
+    def _all(self, condition, entry):
+        return all(self._match(item, entry) for item in condition)
+
+    def _elemMatch(self, condition, entry):
+        if not isinstance(entry, Sequence):
+            return False
+        return any(
+            all(
+                self._process_condition(sub_operator, sub_condition, element)
+                for sub_operator, sub_condition in condition.items()
+            )
+            for element in entry
+        )
+
+    @staticmethod
+    def _size(condition, entry):
+        if not isinstance(condition, int):
+            raise QueryError(
+                f"$size has been attributed incorrect argument {condition}"
+            )
+
+        if is_non_string_sequence(entry):
+            return len(entry) == condition
+
+        return False
+
+    def __repr__(self):
+        return f"<Query({self._definition})>"
+
+
+def toregex(wildcard):
+    if not isinstance(wildcard, str):
+        raise QueryError(f"Unexpected value in the regex conversion: {wildcard}")
+    # If is not neessary, but we avoid unnecessary regular expressions.
+    if any(c in wildcard for c in "?*[]!"):
+        return {"$regex": fnmatch.translate(wildcard)}
+    return wildcard
+
+
+def convert(filters):
+    """Convert filters to the mongo query syntax.
+    A special care is taken to handle old-style negative filters correctly
+    """
+    # Negative_filters are old-style negative assertions, and behave differently.
+    # See issue #246 for the original bug report.
+    #
+    # For a short example:
+    # [{"platform": "!win32"}, {"platform": "!linux"}]
+    # will match all non-linux non-windows samples, but:
+    # [{"platform": {"$not": "win32"}}, {"platform": {"$not": "linux"}}]
+    # means `platform != "win32" or "platform != "linux"` and will match everything.
+    # To get equivalent behaviour with mongo syntax, you should use:
+    # [{"platform": {"$not": {"$or": ["win32", "linux"]}}}]
+    regular_filter, negative_filter = [], []
+    for rule in filters:
+        positive_checks, negative_checks = [], []
+        for key, value in rule.items():
+            if isinstance(value, str):
+                if value and value[0] == "!":  # negative check
+                    negative_checks.append({key: toregex(value[1:])})
+                else:
+                    positive_checks.append({key: toregex(value)})
+            else:
+                positive_checks.append({key: value})
+        regular_filter.append({"$and": positive_checks})
+        negative_filter.append({"$and": positive_checks + [{"$or": negative_checks}]})
+    return Query(
+        {
+            "$and": [
+                {"$not": {"$or": negative_filter}},
+                {"$or": regular_filter},
+            ]
+        }
+    )

karton/core/task.py

@@ -1,5 +1,4 @@
 import enum
-import fnmatch
 import json
 import time
 import uuid
@@ -16,6 +15,7 @@ from typing import (
     Union,
 )
 
+from . import query
 from .resource import RemoteResource, ResourceBase
 from .utils import recursive_iter, recursive_iter_with_keys, recursive_map
 
@@ -223,75 +223,8 @@ class Task(object):
         return new_task
 
     def matches_filters(self, filters: List[Dict[str, Any]]) -> bool:
-        """
-        Checks whether provided task headers match filters
-
-        :param filters: Task header filters
-        :return: True if task headers match specific filters
-
-        :meta private:
-        """
-
-        def test_filter(headers: Dict[str, Any], filter: Dict[str, Any]) -> int:
-            """
-            Filter match follows AND logic, but it's non-boolean because filters may be
-            negated (task:!platform).
-
-            Result values are as follows:
-            - 1  - positive match, no mismatched values in headers
-                   (all matched)
-            - 0  - no match, found value that doesn't match to the filter
-                   (some are not matched)
-            - -1 - negative match, found value that matches negated filter value
-                   (all matched but found negative matches)
-            """
-            matches = 1
-            for filter_key, filter_value in filter.items():
-                # Coerce filter value to string
-                filter_value_str = str(filter_value)
-                negated = False
-                if filter_value_str.startswith("!"):
-                    negated = True
-                    filter_value_str = filter_value_str[1:]
-
-                # If expected key doesn't exist in headers
-                if filter_key not in headers:
-                    # Negated filter ignores non-existent values
-                    if negated:
-                        continue
-                    # But positive filter doesn't
-                    return 0
-
-                # Coerce header value to string
-                header_value_str = str(headers[filter_key])
-                # fnmatch is great for handling simple wildcard patterns (?, *, [abc])
-                match = fnmatch.fnmatchcase(header_value_str, filter_value_str)
-                # If matches, but it's negated: it's negative match
-                if match and negated:
-                    matches = -1
-                # If doesn't match but filter is not negated: it's not a match
-                if not match and not negated:
-                    return 0
-            # If there are no mismatched values: filter is matched
-            return matches
-
-        # List of filter matches follow OR logic, but -1 is special
-        # If there is any -1, result is False
-        #   (any matched, but it's negative match)
-        # If there is any 1, but no -1's: result is True
-        #   (any matched, no negative match)
-        # If there are only 0's: result is False
-        #   (none matched)
-        matches = False
-        for task_filter in filters:
-            match_result = test_filter(self.headers, task_filter)
-            if match_result == -1:
-                # Any negative match results in False
-                return False
-            if match_result == 1:
-                # Any positive match but without negative matches results in True
-                matches = True
-        return matches
+        """Check if a task matches the given filters"""
+        return query.convert(filters).match(self.headers)
 
     def set_task_parent(self, parent: "Task"):
         """
@@ -475,7 +408,12 @@ class Task(object):
         if parse_resources:
             task_data = json.loads(data, object_hook=unserialize_resources)
         else:
-            task_data = orjson.loads(data)
+            try:
+                task_data = orjson.loads(data)
+            except orjson.JSONDecodeError:
+                # Fallback, in case orjson raises exception during loading
+                # This may happen for large numbers (too large for float)
+                task_data = json.loads(data, object_hook=unserialize_resources)
 
         # Compatibility with Karton <5.2.0
         headers_persistent_fallback = task_data["payload_persistent"].get(

karton/system/system.py

@@ -3,6 +3,7 @@ import json
 import time
 from typing import List, Optional
 
+from karton.core import query
 from karton.core.__version__ import __version__
 from karton.core.backend import (
     KARTON_OPERATIONS_QUEUE,
@@ -175,7 +176,12 @@ class SystemService(KartonServiceBase):
         pipe = self.backend.make_pipeline()
         for bind in binds:
             identity = bind.identity
-            if task.matches_filters(bind.filters):
+            try:
+                is_match = task.matches_filters(bind.filters)
+            except query.QueryError:
+                self.log.error("Task matching failed - invalid filters?")
+                continue
+            if is_match:
                 routed_task = task.fork_task()
                 routed_task.status = TaskState.SPAWNED
                 routed_task.last_update = time.time()

{karton_core-5.4.0.dist-info → karton_core-5.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: karton-core
-Version: 5.4.0
+Version: 5.5.1
 Summary: Distributed malware analysis orchestration framework
 Home-page: https://github.com/CERT-Polska/karton
 License: UNKNOWN

{karton_core-5.4.0.dist-info → karton_core-5.5.1.dist-info}/RECORD

@@ -1,26 +1,27 @@
-karton_core-5.4.0-nspkg.pth,sha256=vHa-jm6pBTeInFrmnsHMg9AOeD88czzQy-6QCFbpRcM,539
+karton_core-5.5.1-nspkg.pth,sha256=vHa-jm6pBTeInFrmnsHMg9AOeD88czzQy-6QCFbpRcM,539
 karton/core/__init__.py,sha256=QuT0BWZyp799eY90tK3H1OD2hwuusqMJq8vQwpB3kG4,337
-karton/core/__version__.py,sha256=xjYaBGUFGg0kGZj_WhuoFyPD8NILPsr79SaMwmYQGSg,22
+karton/core/__version__.py,sha256=o0RHk7avNRUho2u_PgLKAFJKJks95Wx54GlOzI6Jzq4,22
 karton/core/backend.py,sha256=-sQG7utnaWLJOEcafeSwEDLnkflPqtSCwg_mn_nnFhg,36727
 karton/core/base.py,sha256=C6Lco3E0XCsxvEjeVOLR9fxh_IWJ1vjC9BqUYsQyewE,8083
 karton/core/config.py,sha256=7oKchitq6pWzPuXRfjBXqVT_BgGIz2p-CDo1RGaNJQg,8118
 karton/core/exceptions.py,sha256=8i9WVzi4PinNlX10Cb-lQQC35Hl-JB5R_UKXa9AUKoQ,153
 karton/core/inspect.py,sha256=aIJQEOEkD5q2xLlV8nhxY5qL5zqcnprP-2DdP6ecKlE,6150
-karton/core/karton.py,sha256=9SOAviG42kSsPqc3EuaHzWtA_KywMtc01hmU6FaJpHo,15007
+karton/core/karton.py,sha256=cXLleTEPCVBIXkj09kKu2hjd1XNUSpTAk87-BES1WlA,15133
 karton/core/logger.py,sha256=J3XAyG88U0cwYC9zR6E3QD1uJenrQh7zS9-HgxhqeAs,2040
 karton/core/main.py,sha256=ir1-dhn3vbwfh2YHiM6ZYfRBbjwLvJSz0d8tuK1mb_4,8310
 karton/core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+karton/core/query.py,sha256=Ay0VzfrBQwdJzcZ27JbOlUc1ZZdOl6A8sh4iIYTmLyE,11493
 karton/core/resource.py,sha256=tA3y_38H9HVKIrCeAU70zHUkQUv0BuCQWMC470JLxxc,20321
-karton/core/task.py,sha256=diwg8uUl57NEYNRjT1l5CPiNw3EQcU11BnrLul33fx0,21350
+karton/core/task.py,sha256=gW1szMi5PN2Y06X-Ryo7cmEVluZv1r7W5tvmwIJiD94,18808
 karton/core/test.py,sha256=tms-YM7sUKQDHN0vm2_W7DIvHnO_ld_VPsWHnsbKSfk,9102
 karton/core/utils.py,sha256=sEVqGdVPyYswWuVn8wYXBQmln8Az826N_2HgC__pmW8,4090
 karton/system/__init__.py,sha256=JF51OqRU_Y4c0unOulvmv1KzSHSq4ZpXU8ZsH4nefRM,63
 karton/system/__main__.py,sha256=QJkwIlSwaPRdzwKlNmCAL41HtDAa73db9MZKWmOfxGM,56
-karton/system/system.py,sha256=yF_d71a8w7JYA7IXUt63d5_QBH6x1QplB-xcrzQTXL4,13792
-karton_core-5.4.0.dist-info/LICENSE,sha256=o8h7hYhn7BJC_-DmrfqWwLjaR_Gbe0TZOOQJuN2ca3I,1519
-karton_core-5.4.0.dist-info/METADATA,sha256=kopeYFCI9EoFQbc7J7woZWjI_5egy29-lYUW7UzEQ2I,6847
-karton_core-5.4.0.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-karton_core-5.4.0.dist-info/entry_points.txt,sha256=FJj5EZuvFP0LkagjX_dLbRGBUnuLjgBiSyiFfq4c86U,99
-karton_core-5.4.0.dist-info/namespace_packages.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
-karton_core-5.4.0.dist-info/top_level.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
-karton_core-5.4.0.dist-info/RECORD,,
+karton/system/system.py,sha256=tptar24RuXUnlII1xKbuJtfNkQsSxTtS3g4O8S99tbg,14011
+karton_core-5.5.1.dist-info/LICENSE,sha256=o8h7hYhn7BJC_-DmrfqWwLjaR_Gbe0TZOOQJuN2ca3I,1519
+karton_core-5.5.1.dist-info/METADATA,sha256=trLfddTECFmbcuReIoSF9Yb9ug1BSAxHBUAM0nNoXnE,6847
+karton_core-5.5.1.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+karton_core-5.5.1.dist-info/entry_points.txt,sha256=FJj5EZuvFP0LkagjX_dLbRGBUnuLjgBiSyiFfq4c86U,99
+karton_core-5.5.1.dist-info/namespace_packages.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
+karton_core-5.5.1.dist-info/top_level.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
+karton_core-5.5.1.dist-info/RECORD,,