karton-core 5.5.1__py3-none-any.whl → 5.6.1__py3-none-any.whl

karton/core/__version__.py

@@ -1 +1 @@
-__version__ = "5.5.1"
+__version__ = "5.6.1"

karton/core/backend.py

@@ -986,6 +986,21 @@ class KartonBackend:
                 objs.append(obj["Key"])
         return objs
 
+    def list_object_versions(self, bucket: str) -> Dict[str, List[str]]:
+        """
+        List version identifiers of stored resource objects
+        :param bucket: Bucket name
+        :return: Dictionary of object version identifiers {key: [version_ids, ...]}
+        """
+        objs = defaultdict(list)
+        paginator = self.s3.get_paginator("list_object_versions")
+        for page in paginator.paginate(Bucket=bucket):
+            for obj in page.get("Versions", list()):
+                objs[obj["Key"]].append(obj["VersionId"])
+            for obj in page.get("DeleteMarkers", list()):
+                objs[obj["Key"]].append(obj["VersionId"])
+        return dict(objs)
+
     def remove_object(self, bucket: str, object_uid: str) -> None:
         """
         Remove resource object from object storage
@@ -1002,7 +1017,38 @@ class KartonBackend:
         :param bucket: Bucket name
         :param object_uids: Object identifiers
         """
-        for delete_objects in chunks([{"Key": uid} for uid in object_uids], 1000):
+        for delete_objects in chunks([{"Key": uid} for uid in object_uids], 100):
+            self.s3.delete_objects(Bucket=bucket, Delete={"Objects": delete_objects})
+
+    def remove_object_versions(
+        self,
+        bucket: str,
+        object_versions: Dict[str, List[str]],
+        explicit_version_null: bool = False,
+    ) -> None:
+        """
+        Bulk remove resource object versions from object storage
+
+        :param bucket: Bucket name
+        :param object_versions: Object version identifiers
+        :param explicit_version_null: |
+            Some S3 providers (e.g. MinIO) need a reference
+            to "null" version explicitly when versioning is in suspended state. On the
+            other hand, some providers refuse to delete "null" versions when bucket
+            versioning is disabled.
+            See also: https://github.com/CERT-Polska/karton/issues/273.
+        """
+        deletion_chunks = chunks(
+            [
+                {"Key": uid, "VersionId": version_id}
+                if version_id != "null" or explicit_version_null
+                else {"Key": uid}
+                for uid, versions in object_versions.items()
+                for version_id in versions
+            ],
+            100,
+        )
+        for delete_objects in deletion_chunks:
             self.s3.delete_objects(Bucket=bucket, Delete={"Objects": delete_objects})
 
     def check_bucket_exists(self, bucket: str, create: bool = False) -> bool:

karton/core/karton.py

@@ -106,11 +106,15 @@ class Consumer(KartonServiceBase):
     :param config: Karton config to use for service configuration
     :param identity: Karton service identity
     :param backend: Karton backend to use
+    :param task_timeout: The maximum time, in seconds, this consumer will wait for
+                         a task to finish processing before being CRASHED on timeout.
+                         Set 0 for unlimited, and None for using global value
     """
 
     filters: List[Dict[str, Any]] = []
     persistent: bool = True
     version: Optional[str] = None
+    task_timeout = None
 
     def __init__(
         self,
@@ -130,7 +134,8 @@ class Consumer(KartonServiceBase):
             self.config.getboolean("karton", "persistent", self.persistent)
             and not self.debug
         )
-        self.task_timeout = self.config.getint("karton", "task_timeout")
+        if self.task_timeout is None:
+            self.task_timeout = self.config.getint("karton", "task_timeout")
         self.current_task: Optional[Task] = None
         self._pre_hooks: List[Tuple[Optional[str], Callable[[Task], None]]] = []
         self._post_hooks: List[
@@ -323,7 +328,7 @@ class Consumer(KartonServiceBase):
         """
         self.log.info("Service %s started", self.identity)
 
-        if self.task_timeout is not None:
+        if self.task_timeout:
             self.log.info(f"Task timeout is set to {self.task_timeout} seconds")
 
         # Get the old binds and set the new ones atomically

karton/core/query.py

@@ -25,8 +25,15 @@ def is_non_string_sequence(entry):
 class Query(object):
     """The Query class is used to match an object against a MongoDB-like query"""
 
-    def __init__(self, definition):
+    def __init__(self, definition, _type_coercion=False):
+        """
+        If _type_coercion is enabled: header values are coerced to string
+        when condition is also a string. It's implemented for compatibility
+        with old syntax e.g. {"execute": "!False"} filter vs {"execute": False}
+        header value.
+        """
         self._definition = definition
+        self._type_coercion = _type_coercion
 
     def match(self, entry):
         """Matches the entry object against the query specified on instanciation"""
@@ -40,7 +47,7 @@ class Query(object):
             )
         if is_non_string_sequence(entry):
             return condition in entry
-        return condition == entry
+        return self._eq(condition, entry)
 
     def _extract(self, entry, path):
         if not path:
@@ -108,9 +115,14 @@ class Query(object):
     def _noop(*_):
         return True
 
-    @staticmethod
-    def _eq(condition, entry):
+    def _eq(self, condition, entry):
         try:
+            if (
+                self._type_coercion
+                and type(condition) is str
+                and type(entry) is not str
+            ):
+                return str(entry) == condition
             return entry == condition
         except TypeError:
             return False
@@ -155,9 +167,8 @@ class Query(object):
         except TypeError:
             return False
 
-    @staticmethod
-    def _ne(condition, entry):
-        return entry != condition
+    def _ne(self, condition, entry):
+        return not self._eq(condition, entry)
 
     def _nin(self, condition, entry):
         return not self._in(condition, entry)
@@ -346,5 +357,6 @@ def convert(filters):
                 {"$not": {"$or": negative_filter}},
                 {"$or": regular_filter},
             ]
-        }
+        },
+        _type_coercion=True,
     )

karton/system/system.py

@@ -26,6 +26,7 @@ class SystemService(KartonServiceBase):
     version = __version__
     with_service_info = True
 
+    CRASH_STARTED_TASKS_ON_TIMEOUT = False
     GC_INTERVAL = 3 * 60
     TASK_DISPATCHED_TIMEOUT = 24 * 3600
     TASK_STARTED_TIMEOUT = 24 * 3600
@@ -45,13 +46,40 @@ class SystemService(KartonServiceBase):
         )
         self.enable_gc = self.config.getboolean("system", "enable_gc", True)
         self.enable_router = self.config.getboolean("system", "enable_router", True)
+        self.crash_started_tasks_on_timeout = self.config.getboolean(
+            "system", "crash_started_tasks_on_timeout", False
+        )
+        self.enable_null_version_deletion = self.config.getboolean(
+            "system", "enable_null_version_deletion", False
+        )
 
         self.last_gc_trigger = time.time()
 
+    def _log_config(self):
+        self.log.info(
+            "Effective config:\n"
+            " gc_interval:\t%s\n"
+            " task_dispatched_timeout:\t%s\n"
+            " task_started_timeout:\t%s\n"
+            " task_crashed_timeout:\t%s\n"
+            " enable_gc:\t%s\n"
+            " enable_router:\t%s\n"
+            " enable_null_version_deletion:\t%s\n"
+            " crash_started_tasks_on_timeout:\t%s",
+            self.gc_interval,
+            self.task_dispatched_timeout,
+            self.task_started_timeout,
+            self.task_crashed_timeout,
+            self.enable_gc,
+            self.enable_router,
+            self.enable_null_version_deletion,
+            self.crash_started_tasks_on_timeout,
+        )
+
     def gc_collect_resources(self) -> None:
         # Collects unreferenced resources left in object storage
         karton_bucket = self.backend.default_bucket_name
-        resources_to_remove = set(self.backend.list_objects(karton_bucket))
+        resources_to_remove = self.backend.list_object_versions(karton_bucket)
         # Note: it is important to get list of resources before getting list of tasks!
         # Task is created before resource upload to lock the reference to the resource.
         tasks = self.backend.iter_all_tasks()
@@ -62,12 +90,17 @@ class SystemService(KartonServiceBase):
                     resource.bucket == karton_bucket
                     and resource.uid in resources_to_remove
                 ):
-                    resources_to_remove.remove(resource.uid)
+                    del resources_to_remove[resource.uid]
         # Remove unreferenced resources
         if resources_to_remove:
-            self.backend.remove_objects(karton_bucket, resources_to_remove)
+            self.backend.remove_object_versions(
+                karton_bucket,
+                resources_to_remove,
+                explicit_version_null=self.enable_null_version_deletion,
+            )
 
     def gc_collect_tasks(self) -> None:
+        self.log.debug("GC: gc_collect_tasks started")
         # Collects finished tasks
         root_tasks = set()
         running_root_tasks = set()
@@ -75,6 +108,7 @@ class SystemService(KartonServiceBase):
 
         current_time = time.time()
         to_delete = []
+        to_crash = []
 
         queues_to_clear = set()
         online_consumers = self.backend.get_online_consumers()
@@ -116,14 +150,24 @@ class SystemService(KartonServiceBase):
                 and task.last_update is not None
                 and current_time > task.last_update + self.task_started_timeout
             ):
-                to_delete.append(task)
-                self.log.error(
-                    "Task %s is in Started state more than %d seconds. "
-                    "Killed. (receiver: %s)",
-                    task.uid,
-                    self.task_started_timeout,
-                    task.headers.get("receiver", "<unknown>"),
-                )
+                if self.crash_started_tasks_on_timeout:
+                    to_crash.append(task)
+                    self.log.error(
+                        "Task %s is in Started state more than %d seconds. "
+                        "Crashed. (receiver: %s)",
+                        task.uid,
+                        self.task_started_timeout,
+                        task.headers.get("receiver", "<unknown>"),
+                    )
+                else:
+                    to_delete.append(task)
+                    self.log.error(
+                        "Task %s is in Started state more than %d seconds. "
+                        "Killed. (receiver: %s)",
+                        task.uid,
+                        self.task_started_timeout,
+                        task.headers.get("receiver", "<unknown>"),
+                    )
             elif task.status == TaskState.FINISHED:
                 to_delete.append(task)
                 self.log.debug("GC: Finished task %s", task.uid)
@@ -151,11 +195,26 @@ class SystemService(KartonServiceBase):
             self.backend.increment_metrics_list(
                 KartonMetrics.TASK_GARBAGE_COLLECTED, to_increment
             )
+        if to_crash:
+            to_increment = [
+                task.headers.get("receiver", "unknown") for task in to_crash
+            ]
+            for task in to_crash:
+                task.error = [
+                    "This task was STARTED too long (TASK_STARTED_TIMEOUT), "
+                    "so status was changes to CRASHED."
+                ]
+                self.backend.set_task_status(task, TaskState.CRASHED)
+            self.backend.increment_metrics_list(
+                KartonMetrics.TASK_CRASHED, to_increment
+            )
 
         for finished_root_task in root_tasks.difference(running_root_tasks):
             # TODO: Notification needed
             self.log.debug("GC: Finished root task %s", finished_root_task)
 
+        self.log.debug("GC: gc_collect_tasks ended")
+
     def gc_collect(self) -> None:
         if time.time() > (self.last_gc_trigger + self.gc_interval):
             try:
@@ -251,6 +310,7 @@ class SystemService(KartonServiceBase):
                 self.handle_operations(bodies)
 
     def loop(self) -> None:
+        self._log_config()
         self.log.info("Manager %s started", self.identity)
 
         with self.graceful_killer():
@@ -288,7 +348,6 @@ class SystemService(KartonServiceBase):
         parser.add_argument(
             "--gc-interval",
             type=int,
-            default=cls.GC_INTERVAL,
             help="Garbage collection interval",
         )
         parser.add_argument(
@@ -304,16 +363,24 @@ class SystemService(KartonServiceBase):
         parser.add_argument(
             "--task-crashed-timeout", help="Timeout for tasks in Crashed state"
         )
+        parser.add_argument(
+            "--crash-started-task-on-timeout",
+            action="store_const",
+            dest="crash_started_tasks",
+            help="Crash Started tasks on timeout instead of deleting",
+        )
         return parser
 
     @classmethod
     def config_from_args(cls, config: Config, args: argparse.Namespace):
         super().config_from_args(config, args)
+
         config.load_from_dict(
             {
                 "system": {
                     "enable_gc": args.enable_gc,
                     "enable_router": args.enable_router,
+                    "crash_started_tasks_on_timeout": args.crash_started_tasks,
                     "gc_interval": args.gc_interval,
                     "task_dispatched_timeout": args.task_dispatched_timeout,
                     "task_started_timeout": args.task_started_timeout,

{karton_core-5.5.1.dist-info → karton_core-5.6.1.dist-info}/METADATA

@@ -1,17 +1,15 @@
 Metadata-Version: 2.1
 Name: karton-core
-Version: 5.5.1
+Version: 5.6.1
 Summary: Distributed malware analysis orchestration framework
 Home-page: https://github.com/CERT-Polska/karton
-License: UNKNOWN
-Platform: UNKNOWN
 Classifier: Programming Language :: Python :: 3
 Classifier: Operating System :: OS Independent
 Classifier: License :: OSI Approved :: BSD License
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: boto3
+Requires-Dist: boto3 <1.36.0
 Requires-Dist: orjson
 Requires-Dist: redis
 
@@ -141,5 +139,3 @@ RetDec unpacker module for the Karton framework
 Malware similarity platform with modularity in mind.
 
 ![Co-financed by the Connecting Europe Facility by of the European Union](https://www.cert.pl/uploads/2019/02/en_horizontal_cef_logo-e1550495232540.png)
-
-

{karton_core-5.5.1.dist-info → karton_core-5.6.1.dist-info}/RECORD

@@ -1,27 +1,27 @@
-karton_core-5.5.1-nspkg.pth,sha256=vHa-jm6pBTeInFrmnsHMg9AOeD88czzQy-6QCFbpRcM,539
+karton_core-5.6.1-nspkg.pth,sha256=vHa-jm6pBTeInFrmnsHMg9AOeD88czzQy-6QCFbpRcM,539
 karton/core/__init__.py,sha256=QuT0BWZyp799eY90tK3H1OD2hwuusqMJq8vQwpB3kG4,337
-karton/core/__version__.py,sha256=o0RHk7avNRUho2u_PgLKAFJKJks95Wx54GlOzI6Jzq4,22
-karton/core/backend.py,sha256=-sQG7utnaWLJOEcafeSwEDLnkflPqtSCwg_mn_nnFhg,36727
+karton/core/__version__.py,sha256=-q9tSF5ofTJum4PMjvbhaE1xmTXehc_9rxMGcmfodcw,22
+karton/core/backend.py,sha256=g0BSQBsFAksRd_VY5QDjBJ8yIIyzAmwxy-kfJgAZ_lo,38628
 karton/core/base.py,sha256=C6Lco3E0XCsxvEjeVOLR9fxh_IWJ1vjC9BqUYsQyewE,8083
 karton/core/config.py,sha256=7oKchitq6pWzPuXRfjBXqVT_BgGIz2p-CDo1RGaNJQg,8118
 karton/core/exceptions.py,sha256=8i9WVzi4PinNlX10Cb-lQQC35Hl-JB5R_UKXa9AUKoQ,153
 karton/core/inspect.py,sha256=aIJQEOEkD5q2xLlV8nhxY5qL5zqcnprP-2DdP6ecKlE,6150
-karton/core/karton.py,sha256=cXLleTEPCVBIXkj09kKu2hjd1XNUSpTAk87-BES1WlA,15133
+karton/core/karton.py,sha256=Fi3wNqMGiKvHN2BECsqsvfxkiyuwPdlC21jpqQdkeak,15434
 karton/core/logger.py,sha256=J3XAyG88U0cwYC9zR6E3QD1uJenrQh7zS9-HgxhqeAs,2040
 karton/core/main.py,sha256=ir1-dhn3vbwfh2YHiM6ZYfRBbjwLvJSz0d8tuK1mb_4,8310
 karton/core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-karton/core/query.py,sha256=Ay0VzfrBQwdJzcZ27JbOlUc1ZZdOl6A8sh4iIYTmLyE,11493
+karton/core/query.py,sha256=sf24DweVlXfJuBbBD_ns2LXhOV-IBwuPG3jBfTJu77s,12063
 karton/core/resource.py,sha256=tA3y_38H9HVKIrCeAU70zHUkQUv0BuCQWMC470JLxxc,20321
 karton/core/task.py,sha256=gW1szMi5PN2Y06X-Ryo7cmEVluZv1r7W5tvmwIJiD94,18808
 karton/core/test.py,sha256=tms-YM7sUKQDHN0vm2_W7DIvHnO_ld_VPsWHnsbKSfk,9102
 karton/core/utils.py,sha256=sEVqGdVPyYswWuVn8wYXBQmln8Az826N_2HgC__pmW8,4090
 karton/system/__init__.py,sha256=JF51OqRU_Y4c0unOulvmv1KzSHSq4ZpXU8ZsH4nefRM,63
 karton/system/__main__.py,sha256=QJkwIlSwaPRdzwKlNmCAL41HtDAa73db9MZKWmOfxGM,56
-karton/system/system.py,sha256=tptar24RuXUnlII1xKbuJtfNkQsSxTtS3g4O8S99tbg,14011
-karton_core-5.5.1.dist-info/LICENSE,sha256=o8h7hYhn7BJC_-DmrfqWwLjaR_Gbe0TZOOQJuN2ca3I,1519
-karton_core-5.5.1.dist-info/METADATA,sha256=trLfddTECFmbcuReIoSF9Yb9ug1BSAxHBUAM0nNoXnE,6847
-karton_core-5.5.1.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-karton_core-5.5.1.dist-info/entry_points.txt,sha256=FJj5EZuvFP0LkagjX_dLbRGBUnuLjgBiSyiFfq4c86U,99
-karton_core-5.5.1.dist-info/namespace_packages.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
-karton_core-5.5.1.dist-info/top_level.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
-karton_core-5.5.1.dist-info/RECORD,,
+karton/system/system.py,sha256=cFE4hCS0LWnwdCiIjU0ym8dHujE5ORi4REJR_y5b2gA,16671
+karton_core-5.6.1.dist-info/LICENSE,sha256=o8h7hYhn7BJC_-DmrfqWwLjaR_Gbe0TZOOQJuN2ca3I,1519
+karton_core-5.6.1.dist-info/METADATA,sha256=AJoa9O_0SOYI3IuVHXhwB6lXoUSs7S4nU6QM8_xHxVI,6818
+karton_core-5.6.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+karton_core-5.6.1.dist-info/entry_points.txt,sha256=OgLlsXy61GP6-Yob3oXqeJ2hlRU6LBLj33fr0NufKz0,98
+karton_core-5.6.1.dist-info/namespace_packages.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
+karton_core-5.6.1.dist-info/top_level.txt,sha256=X8SslCPsqXDCnGZqrYYolzT3xPzJMq1r-ZQSc0jfAEA,7
+karton_core-5.6.1.dist-info/RECORD,,

{karton_core-5.5.1.dist-info → karton_core-5.6.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.37.1)
+Generator: bdist_wheel (0.42.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{karton_core-5.5.1.dist-info → karton_core-5.6.1.dist-info}/entry_points.txt

@@ -1,4 +1,3 @@
 [console_scripts]
 karton = karton.core.main:main
 karton-system = karton.system:SystemService.main
-